From 67888ab3b265a64910aaf407b96740e68cefcf9c Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Wed, 17 Mar 2021 15:05:50 -0400
Subject: [PATCH 01/27] - matching collection functionality to Gallon 1.1.16 -
 show last price - show last acquisition date - show acquisition action

---
 conseilUtil.js | 101 +++++++++++++++++++++++++++++++++++++++++--------
 package.json   |   1 +
 2 files changed, 87 insertions(+), 15 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index ddcb89d..673ba0b 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -1,6 +1,7 @@
 const conseiljs = require('conseiljs')
 const fetch = require('node-fetch')
 const log = require('loglevel')
+const BigNumber = require('bignumber.js')
 
 const logger = log.getLogger('conseiljs')
 logger.setLevel('error', false)
@@ -20,7 +21,7 @@ const mainnet = require('./config').networkConfig
  */
 const getCollectionForAddress = async (address) => {
     let collectionQuery = conseiljs.ConseilQueryBuilder.blankQuery();
-    collectionQuery = conseiljs.ConseilQueryBuilder.addFields(collectionQuery, 'key', 'value');
+    collectionQuery = conseiljs.ConseilQueryBuilder.addFields(collectionQuery, 'key', 'value', 'operation_group_id');
     collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger])
     collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'key', conseiljs.ConseilOperator.STARTSWITH, [
         `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)}`,
@@ -30,10 +31,14 @@ const getCollectionForAddress = async (address) => {
 
     const collectionResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', collectionQuery);
     let collection = collectionResult.map((i) => {
-        return { piece: i.key.toString().replace(/.* ([0-9]{1,}$)/, '$1'), amount: Number(i.value) }
+        return {
+            piece: i['key'].toString().replace(/.* ([0-9]{1,}$)/, '$1'),
+            amount: Number(i['value']),
+            opId: i['operation_group_id']
+        }
     })
 
-    const queryChunks = chunkArray(collection.map(i => i.piece), 20) // NOTE: consider increasing this number somewhat
+    const queryChunks = chunkArray(collection.map(i => i.piece), 50) // NOTE: consider increasing this number somewhat
     const makeObjectQuery = (keys) => {
         let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
         mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value');
@@ -55,17 +60,84 @@ const getCollectionForAddress = async (address) => {
             objectIpfsMap[objectId] = ipfsHash
     }))))
 
-    collection = collection.map(i => { return {
-        ipfsHash: objectIpfsMap[i.piece.toString()],
-        ...i
+    const operationGroupIds = collectionResult.map((r) => r.operation_group_id)
+    const priceQueryChunks = chunkArray(operationGroupIds, 30)
+    const makeLastPriceQuery = (opIds) => {
+        let lastPriceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.addFields(lastPriceQuery, 'timestamp', 'amount', 'operation_group_hash', 'parameters_entrypoints', 'parameters');
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']);
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']);
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'internal', conseiljs.ConseilOperator.EQ, ['false']);
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'operation_group_hash', opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ, opIds);
+        lastPriceQuery = conseiljs.ConseilQueryBuilder.setLimit(lastPriceQuery, opIds.length);
+
+        return lastPriceQuery;
+    }
+
+    const priceQueries = priceQueryChunks.map((c) => makeLastPriceQuery(c))
+    const priceMap = {};
+    await Promise.all(
+        priceQueries.map(
+            async (q) =>
+                await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'operations', q).then((result) =>
+                    result.map((row) => {
+                        let amount = 0;
+                        const action = row.parameters_entrypoints;
+
+                        if (action === 'collect') {
+                            amount = Number(row.parameters.toString().replace(/^Pair ([0-9]+) [0-9]+/, '$1'));
+                        } else if (action === 'transfer') {
+                            amount = Number(
+                                row.parameters
+                                    .toString()
+                                    .replace(
+                                        /[{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [(]Pair [0-9]+ [0-9]+[)] [}] [}]/,
+                                        '$1'
+                                    )
+                            );
+                        }
+
+                        priceMap[row.operation_group_hash] = {
+                            price: new BigNumber(row.amount),
+                            amount,
+                            timestamp: row.timestamp,
+                            action,
+                        };
+                    })
+                )
+        )
+    )
+
+    collection = collection.map(i => {
+        let price = 0
+        let receivedOn = new Date()
+        let action = ''
+
+        try {
+            const priceRecord = priceMap[i.opId]
+            price = priceRecord.price.dividedToIntegerBy(priceRecord.amount).toNumber()
+            receivedOn = new Date(priceRecord.timestamp)
+            action = priceRecord.action === 'collect' ? 'Purchased' : 'Received'
+        } catch {
+            //
+        }
+
+        delete i.opId
+
+        return {
+            price: isNaN(price) ? 0 : price,
+            receivedOn,
+            action,
+            ipfsHash: objectIpfsMap[i.piece.toString()],
+            ...i
     }})
 
     return collection.sort((a, b) => parseInt(b.piece) - parseInt(a.piece)) // sort descending by id – most-recently minted art first
 }
 
 const gethDaoBalanceForAddress = async (address) => {
-    let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
-    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'value');
+    let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery()
+    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'value')
     hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.daoLedger])
     hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [
         `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} 0`
@@ -76,7 +148,7 @@ const gethDaoBalanceForAddress = async (address) => {
     let balance = 0
 
     try {
-        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery);
+        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery)
         balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it
     } catch (error) {
         console.log(`gethDaoBalanceForAddress failed for ${JSON.stringify(hDaoBalanceQuery)} with ${error}`)
@@ -92,8 +164,8 @@ const gethDaoBalanceForAddress = async (address) => {
  * @returns 
  */
 const getArtisticOutputForAddress = async (address) => {
-    let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery();
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash');
+    let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery()
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash')
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction'])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied'])
@@ -107,14 +179,14 @@ const getArtisticOutputForAddress = async (address) => {
         { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },
         'mainnet',
         'operations',
-        mintOperationQuery);
+        mintOperationQuery)
 
     const operationGroupIds = mintOperationResult.map(r => r['operation_group_hash'])
     const queryChunks = chunkArray(operationGroupIds, 30)
 
     const makeObjectQuery = (opIds) => {
-        let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
-        mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value');
+        let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery()
+        mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value')
         mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap])
         mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'operation_group_id', (opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), opIds)
         mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, opIds.length)
@@ -222,7 +294,6 @@ const getArtisticUniverse = async (offset) => {
  * @param {number} objectId 
  * @returns 
  */
-
 const getObjectById = async (objectId) => {
     let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'value');
diff --git a/package.json b/package.json
index 04233cd..5536425 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
   "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
   "dependencies": {
     "axios": "^0.21.1",
+    "bignumber.js": "9.0.1",
     "conseiljs": "5.0.7-2",
     "cors": "^2.8.5",
     "express": "^4.17.1",

From e51c7e99f4dd4c5b837cccb96e617d4c0ebc147e Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Wed, 17 Mar 2021 15:10:35 -0400
Subject: [PATCH 02/27] - sort collection by acquisition date

---
 conseilUtil.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index d25d361..c986026 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -132,7 +132,7 @@ const getCollectionForAddress = async (address) => {
             ...i
     }})
 
-    return collection.sort((a, b) => parseInt(b.piece) - parseInt(a.piece)) // sort descending by id – most-recently minted art first
+    return collection.sort((a, b) => b.receivedOn.getTime() - a.receivedOn.getTime()) // sort descending by date – most-recently acquired art first
 }
 
 const gethDaoBalanceForAddress = async (address) => {

From 67e6a076c3360a89a2a9c65565af67082b5a2af4 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Fri, 19 Mar 2021 02:24:48 -0400
Subject: [PATCH 03/27] rollup

---
 package.json | 71 +++++++++++++++++++++++++++-------------------------
 1 file changed, 37 insertions(+), 34 deletions(-)

diff --git a/package.json b/package.json
index 5536425..527346b 100644
--- a/package.json
+++ b/package.json
@@ -1,35 +1,38 @@
 {
-  "name": "hicetnunc-apiv2",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "start": "node index.js"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/hicetnunc2000/hicetnunc.git"
-  },
-  "author": "@hicetnunc2000",
-  "license": "MIT",
-  "bugs": {
-    "url": "https://github.com/hicetnunc2000/hicetnunc/issues"
-  },
-  "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
-  "dependencies": {
-    "axios": "^0.21.1",
-    "bignumber.js": "9.0.1",
-    "conseiljs": "5.0.7-2",
-    "cors": "^2.8.5",
-    "express": "^4.17.1",
-    "fetch": "^1.1.0",
-    "lodash": "^4.17.21",
-    "loglevel": "1.7.1",
-    "node-fetch": "2.6.1",
-    "serverless-http": "^2.7.0"
-  },
-  "engines": {
-    "node": "12.20.1",
-    "npm": "6.14.10"
-  }
-}
+    "name": "hicetnunc-apiv2",
+    "version": "1.0.0",
+    "description": "",
+    "main": "index.js",
+    "scripts": {
+      "start": "node index.js"
+    },
+    "repository": {
+      "type": "git",
+      "url": "git+https://github.com/hicetnunc2000/hicetnunc.git"
+    },
+    "author": "@hicetnunc2000",
+    "license": "MIT",
+    "bugs": {
+      "url": "https://github.com/hicetnunc2000/hicetnunc/issues"
+    },
+    "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
+    "dependencies": {
+      "axios": "^0.21.1",
+      "bignumber.js": "9.0.1",
+      "cloud-local-storage": "0.0.11",
+      "conseiljs": "5.0.7-2",
+      "cors": "^2.8.5",
+      "dotenv": "^8.2.0",
+      "express": "^4.17.1",
+      "fetch": "^1.1.0",
+      "lodash": "^4.17.21",
+      "loglevel": "1.7.1",
+      "node-fetch": "2.6.1",
+      "serverless-dotenv-plugin": "^3.8.1",
+      "serverless-http": "^2.7.0"
+    },
+    "engines": {
+      "node": "12.20.1",
+      "npm": "6.14.10"
+    }
+  }
\ No newline at end of file

From 020eb2ee73893b2f5aeec556f824598c00f143af Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Fri, 19 Mar 2021 02:25:56 -0400
Subject: [PATCH 04/27] rollup

---
 package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.json b/package.json
index 527346b..d8ce255 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,6 @@
     "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme",
     "dependencies": {
       "axios": "^0.21.1",
-      "bignumber.js": "9.0.1",
       "cloud-local-storage": "0.0.11",
       "conseiljs": "5.0.7-2",
       "cors": "^2.8.5",

From c297de85b08f6fafd8daafcbb1cf64de975d2a57 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Fri, 19 Mar 2021 10:28:29 -0400
Subject: [PATCH 05/27] - increase resultset size

---
 conseilUtil.js | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index e0b6c2d..0c441f9 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -212,12 +212,11 @@ const getArtisticUniverse = async (max_time) => {
     let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash');
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction'])
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.BEFORE, [max_time]) // quantized time (1min)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.BETWEEN, [1612240919000, max_time]) // after 2021 Feb 1
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied'])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT'])
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 7000)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 20_000)
 
     const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData(
         { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },
@@ -230,7 +229,7 @@ const getArtisticUniverse = async (max_time) => {
     let royaltiesQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     royaltiesQuery = conseiljs.ConseilQueryBuilder.addFields(royaltiesQuery, 'key', 'value');
     royaltiesQuery = conseiljs.ConseilQueryBuilder.addPredicate(royaltiesQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftRoyaltiesMap])
-    royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 10_000)
+    royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 20_000)
     const royaltiesResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', royaltiesQuery)
     let artistMap = {}
     royaltiesResult.forEach(row => {
@@ -238,9 +237,9 @@ const getArtisticUniverse = async (max_time) => {
     })
 
     let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
-    swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value');
+    swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value')
     swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap])
-    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 10_000) // NOTE, limited to 10_000
+    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 20_000)
 
     const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery)
 

From 1afcee0d7f98969dd0f99bf85e6ed0a865398f61 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <anonymoussprocket@gmail.com>
Date: Fri, 19 Mar 2021 14:32:39 -0400
Subject: [PATCH 06/27] - crash solution for threaded queries

---
 conseilUtil.js | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index 0c441f9..b022581 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -276,14 +276,21 @@ const getArtisticUniverse = async (max_time) => {
     const objectQueries = queryChunks.map(c => makeObjectQuery(c))
 
     let universe = []
-    await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
-        .then(result => result.map(row => {
-            const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
-            const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
-            const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7);
-
-            universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []})
-    }))))
+    await Promise.all(
+        objectQueries.map(async (q) =>  {
+            const r = []
+            try {
+                r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q)
+                    .then(result => result.map(row => {
+                        const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1')
+                        const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1')
+                        const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7);
+
+                    universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []})
+                    })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected
+            } finally {
+                return r
+            }}))
 
     return universe
 }

From 1cefa8cb1506f41a5dc3faebc1b4ab18e40b95ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 1 Apr 2021 14:08:00 +0200
Subject: [PATCH 07/27] Add simple filtering and rebalancing of feed based on
 creator hDAO Add "all" feed endpoint that doesn't have balancing

Note, this needs to have the threshold changed in a while, for now hDAO is pretty cheap buuuuut
---
 conseilUtil.js | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++
 index.js       | 27 ++++++++++++++----
 2 files changed, 98 insertions(+), 5 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index d3fc4af..b06b24d 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -101,6 +101,67 @@ const gethDaoBalanceForAddress = async (address) => {
     return balance
 }
 
+const gethDaoBalances = async () => {
+    let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'key', 'value');
+    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.daoLedger])
+    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true)
+    hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(hDaoBalanceQuery, 500_000)
+
+    let balance = 0
+    let hdaoMap = {}
+
+    try {
+        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery);
+        
+
+        balanceResult.forEach(row => {
+            hdaoMap[conseiljs.TezosMessageUtils.readAddress(row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1'))] = row['value']
+        })
+        //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it
+    } catch (error) {
+        console.log(`gethDaoBalanceForAddress failed for ${JSON.stringify(hDaoBalanceQuery)} with ${error}`)
+    }
+
+
+    return hdaoMap
+
+}
+
+const getObjektMintingsLastWeek = async () => {
+    var d = new Date();
+    d.setDate(d.getDate()-5);
+    let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'source');
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction'])
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [d.getTime()]) // 2021 Feb 1
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied'])
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol])
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT'])
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 900_000) // TODO: this is hardwired and will not work for highly productive artists
+
+    const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData(
+        { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },
+        'mainnet',
+        'operations',
+        mintOperationQuery);
+
+    const mints = mintOperationResult.map(r => r['source'])
+
+    var initialValue = {}
+    var reducer = function(minters, mintOp) {
+    if (!minters[mintOp]) {
+        minters[mintOp] = 1;
+    } else {
+        minters[mintOp] = minters[mintOp] + 1;
+    }
+    return minters;
+    }
+    return mints.reduce(reducer, initialValue)
+}
+
+
 /**
  * Queries Conseil in two steps to get all the objects minted by a specific address. Step 1 is to query for all 'mint_OBJKT' operations performed by the account to get the list of operation group hashes. Then that list is partitioned into chunks and another query (or set of queries) is run to get big_map values. These values are then parsed into an array of 3-tuples containing the hashed big_map key that can be used to query a Tezos node directly, the nft token id and the ipfs item hash.
  * 
@@ -239,6 +300,20 @@ const getArtisticUniverse = async (max_time) => {
     return universe
 }
 
+
+const getFeaturedArtisticUniverse = async(max_time) => {
+
+    hdaoMap = await gethDaoBalances()
+
+    mintsPerCreator = await getObjektMintingsLastWeek()
+
+    artisticUniverse = await getArtisticUniverse(max_time)
+
+    return artisticUniverse.filter(function (o) {
+        return ((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > 1000000
+    })
+}
+
 /**
  * Returns object ipfs hash and swaps if any
  * 
@@ -304,5 +379,6 @@ module.exports = {
     getArtisticOutputForAddress,
     getObjectById,
     getArtisticUniverse,
+    getFeaturedArtisticUniverse,
     hDAOFeed
 }
diff --git a/index.js b/index.js
index 3b41095..f8e2ba8 100644
--- a/index.js
+++ b/index.js
@@ -153,14 +153,19 @@ const randomFeed = async (counter, res) => {
     })
 }
 
-const getFeed = async (counter, res) => {
+const getFeed = async (counter, res, featured) => {
 
     /*     const now_time = Date.now()
         const immutable = (typeof max_time !== 'undefined') && (max_time < now_time)
         max_time = (typeof max_time !== 'undefined') ? max_time : customFloor(now_time, ONE_MINUTE_MILLIS)
      */
     console.log('feed')
-    var arr = await conseilUtil.getArtisticUniverse(0)
+    var arr
+    if (featured) {
+        arr = await conseilUtil.getFeaturedArtisticUniverse(0)
+    } else {
+        arr = await conseilUtil.getArtisticUniverse(0)
+    }
 
     var feed = offset(desc(arr), counter)
     console.log(feed)
@@ -302,7 +307,19 @@ app.post('/feed', async (req, res) => {
             max_time = null
         } 
     */
-    await getFeed(req.body.counter, res)
+    await getFeed(req.body.counter, res, true)
+})
+
+app.post('/all', async (req, res) => {
+    /*     
+        var counter = req.query.counter
+        var max_time = req.query.hasOwnProperty('time') ? customFloor(req.query.time, ONE_MINUTE_MILLIS) : null
+        const now_time_qt = customFloor(Date.now(), ONE_MINUTE_MILLIS)
+        if (max_time != null & max_time > now_time_qt) {
+            max_time = null
+        } 
+    */
+    await getFeed(req.body.counter, res, false)
 })
 
 app.post('/random', async (req, res) => {
@@ -341,6 +358,6 @@ app.post('/hdao', async (req, res) => {
 const testhdao = async () =>  await hDAOFeed(parseInt(0))
 //testhdao()
 
-//app.listen(3001)
-module.exports.handler = serverless(app)
+app.listen(3001)
+//module.exports.handler = serverless(app)
 

From f502c5dfea4388f2506f590dfccd5b66d337ea3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 1 Apr 2021 14:08:47 +0200
Subject: [PATCH 08/27] Reenable serverless

---
 index.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index f8e2ba8..38eed2e 100644
--- a/index.js
+++ b/index.js
@@ -358,6 +358,6 @@ app.post('/hdao', async (req, res) => {
 const testhdao = async () =>  await hDAOFeed(parseInt(0))
 //testhdao()
 
-app.listen(3001)
-//module.exports.handler = serverless(app)
+//app.listen(3001)
+module.exports.handler = serverless(app)
 

From e7a4588423f398c2055251a8939183f44bd22766 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 1 Apr 2021 14:59:27 +0200
Subject: [PATCH 09/27] Volatility dampening

---
 conseilUtil.js | 49 +++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index b06b24d..9791ee7 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -9,7 +9,7 @@ conseiljs.registerFetch(fetch)
 const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
 const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
 const tezosNode = ''
-
+const quipuSwapContract = "KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph"
 const mainnet = require('./config').networkConfig
 
 
@@ -101,6 +101,37 @@ const gethDaoBalanceForAddress = async (address) => {
     return balance
 }
 
+
+const getTezBalanceForAddress = async (address) => {
+    let accountQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+    accountQuery = conseiljs.ConseilQueryBuilder.addFields(accountQuery, 'balance');
+    accountQuery = conseiljs.ConseilQueryBuilder.addPredicate(accountQuery, 'account_id', conseiljs.ConseilOperator.EQ, [address], false);
+    accountQuery = conseiljs.ConseilQueryBuilder.setLimit(accountQuery, 1);
+
+    try {
+        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'accounts', accountQuery);
+        balance = balanceResult[0]['balance'] // TODO: consider bigNumber here, for the moment there is no reason for it
+    } catch (error) {
+        console.log(`getTezBalanceForAddress failed for ${JSON.stringify(accountQuery)} with ${error}`)
+    }
+
+    return balance
+}
+
+
+const gethDAOPerTez = async() => {
+    const tezBalance = await(getTezBalanceForAddress(quipuSwapContract))
+    const hdaoBalance = await(gethDaoBalanceForAddress(quipuSwapContract))
+    return hdaoBalance / tezBalance
+}
+
+const getTezPerhDAO = async() => {
+    hDAOpTez = await gethDAOPerTez()
+    return 1 / hDAOpTez
+}
+
+
+
 const gethDaoBalances = async () => {
     let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'key', 'value');
@@ -309,8 +340,22 @@ const getFeaturedArtisticUniverse = async(max_time) => {
 
     artisticUniverse = await getArtisticUniverse(max_time)
 
+    hdaoPerTez = await gethDAOPerTez()
+
+    // Cost to be on feed per objekt last 7 days shouldn't be higher than:
+    //   0.1tez
+    //   1 hDAO 
+    // But not lower than:
+    //   0.01 hDAO
+    //
+    // We should probably add more thresholds like $, € and yen
+    // It should be cheap but not too cheap and it shouldn't be
+    // affected by tez or hDAO volatility
+
+    thresholdHdao = Math.min(1_000_000, Math.max(100_000 * hdaoPerTez, 10_000))
+
     return artisticUniverse.filter(function (o) {
-        return ((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > 1000000
+        return ((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao
     })
 }
 

From 89595f3497c9f96290d4de848e1052fdee9261cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Sat, 3 Apr 2021 16:42:22 +0200
Subject: [PATCH 10/27] Add recommended curate amt

---
 config.js      |  5 +++-
 conseilUtil.js | 62 +++++++++++++++++++++++++++++++++++++++++++-------
 index.js       |  5 ++++
 3 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/config.js b/config.js
index 9343184..ccd3cfd 100644
--- a/config.js
+++ b/config.js
@@ -9,7 +9,10 @@ const mainnet = {
     nftSwapMap: 523,
     curationsPtr: 519,
     nftRoyaltiesMap: 522,
-    daoLedger: 515
+    daoLedger: 515,
+    kolibriLedger: 380,
+    hDaoSwap: "KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph", 
+    kolibriSwap: "KT1CiSKXR68qYSxnbzjwvfeMCRburaSDonT2",
 }
 
 module.exports = {
diff --git a/conseilUtil.js b/conseilUtil.js
index 9791ee7..d88e616 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -9,7 +9,6 @@ conseiljs.registerFetch(fetch)
 const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
 const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
 const tezosNode = ''
-const quipuSwapContract = "KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph"
 const mainnet = require('./config').networkConfig
 
 
@@ -101,6 +100,35 @@ const gethDaoBalanceForAddress = async (address) => {
     return balance
 }
 
+const getTokenBalance = async (big_map_id, address, fa2=false, token_id=0) => {
+    let tokenBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+    tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(tokenBalanceQuery, 'value');
+    tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [big_map_id])
+    if (fa2) {
+        tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [
+            `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} ${token_id}`
+        ])
+    }
+    else {
+        tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [
+            `0x${conseiljs.TezosMessageUtils.writeAddress(address)}`
+        ])
+    }
+    tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true)
+    tokenBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(tokenBalanceQuery, 1)
+
+    let balance = 0
+
+    try {
+        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', tokenBalanceQuery);
+        balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it
+    } catch (error) {
+        console.log(`getTokenBalance failed for ${JSON.stringify(tokenBalanceQuery)} with ${error}`)
+    }
+
+    return balance
+}
+
 
 const getTezBalanceForAddress = async (address) => {
     let accountQuery = conseiljs.ConseilQueryBuilder.blankQuery();
@@ -120,18 +148,23 @@ const getTezBalanceForAddress = async (address) => {
 
 
 const gethDAOPerTez = async() => {
-    const tezBalance = await(getTezBalanceForAddress(quipuSwapContract))
-    const hdaoBalance = await(gethDaoBalanceForAddress(quipuSwapContract))
+    const tezBalance = await(getTezBalanceForAddress(mainnet.hDaoSwap))
+    const hdaoBalance = await(gethDaoBalanceForAddress(mainnet.hDaoSwap))
     return hdaoBalance / tezBalance
 }
 
-const getTezPerhDAO = async() => {
-    hDAOpTez = await gethDAOPerTez()
-    return 1 / hDAOpTez
+const getKolibriPerTez = async() => {
+    const tezBalance = await(getTezBalanceForAddress(mainnet.kolibriSwap))
+    //console.log("Tez balance", tezBalance)
+    var kolibriBalance = await(getTokenBalance(mainnet.kolibriLedger, mainnet.kolibriSwap))
+    kolibriBalance = parseInt(kolibriBalance.replace("Pair {} ", "")) / (10**((18 - 6)))
+    //console.log(kolibriBalance)
+    //console.log(typeof(kolibriBalance))
+    //console.log("Kol balance", kolibriBalance)
+    return kolibriBalance / tezBalance
 }
 
 
-
 const gethDaoBalances = async () => {
     let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'key', 'value');
@@ -359,6 +392,18 @@ const getFeaturedArtisticUniverse = async(max_time) => {
     })
 }
 
+const getRecommendedCurateDefault = async() => {
+    hdaoPerTez = await gethDAOPerTez()
+    kolPerTez = await getKolibriPerTez()
+    //console.log("hdaoPerTez", hdaoPerTez)
+    //console.log("kolPerTez", kolPerTez)
+    hdaoPerKol = hdaoPerTez / kolPerTez
+    //console.log("hdaoPerKol", hdaoPerKol)
+
+    //Minimum of $0.1, 0.1 hDAO, and 0.1tez, in hDAO
+    return Math.floor(Math.min(hdaoPerKol * 0.1, 0.1, 0.1 * hdaoPerTez) * 1_000_000)
+}
+
 /**
  * Returns object ipfs hash and swaps if any
  * 
@@ -425,5 +470,6 @@ module.exports = {
     getObjectById,
     getArtisticUniverse,
     getFeaturedArtisticUniverse,
-    hDAOFeed
+    hDAOFeed,
+    getRecommendedCurateDefault
 }
diff --git a/index.js b/index.js
index 38eed2e..bcbb05c 100644
--- a/index.js
+++ b/index.js
@@ -351,6 +351,11 @@ app.post('/objkt', async (req, res) => {
         res.json({ result: await getObjktById(req.body.objkt_id) })
 })
 
+app.get('/recommend_curate', async (req, res) => {
+    const amt = await conseilUtil.getRecommendedCurateDefault()
+    res.json({ amount: amt })
+})
+
 app.post('/hdao', async (req, res) => {
     await hDAOFeed(parseInt(req.body.counter), res)
 })

From 0c2d361ad2626adb480156e16369857d10b44747 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Sat, 3 Apr 2021 19:05:00 +0200
Subject: [PATCH 11/27] Restore /feed to be as it used to be and put new feed
 under /featured

---
 index.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index bcbb05c..6729847 100644
--- a/index.js
+++ b/index.js
@@ -298,7 +298,7 @@ const app = express()
 app.use(express.json())
 app.use(cors({ origin: '*' }))
 
-app.post('/feed', async (req, res) => {
+app.post('/featured', async (req, res) => {
     /*     
         var counter = req.query.counter
         var max_time = req.query.hasOwnProperty('time') ? customFloor(req.query.time, ONE_MINUTE_MILLIS) : null
@@ -310,7 +310,7 @@ app.post('/feed', async (req, res) => {
     await getFeed(req.body.counter, res, true)
 })
 
-app.post('/all', async (req, res) => {
+app.post('/feed', async (req, res) => {
     /*     
         var counter = req.query.counter
         var max_time = req.query.hasOwnProperty('time') ? customFloor(req.query.time, ONE_MINUTE_MILLIS) : null

From 090d62e13f961f39b60de0c6116fb558dffa4ac9 Mon Sep 17 00:00:00 2001
From: mike sizz <mikeiscoding@gmail.com>
Date: Sat, 3 Apr 2021 14:38:27 -0700
Subject: [PATCH 12/27] Sort creations list by objkt ID after removing burned
 objkts.

---
 index.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/index.js b/index.js
index 9092b38..4d3b727 100644
--- a/index.js
+++ b/index.js
@@ -224,6 +224,8 @@ const getTzLedger = async (tz, res) => {
         return arr
     }))
 
+    validCreations = validCreations.sort((a, b) => parseInt(b.objectId) - parseInt(a.objectId))
+
     var arr = []
     console.log([...collection, ...validCreations])
     var arr = [...collection, ...validCreations]

From fb36ecdafa26a97493d89552901b42ad00d5900a Mon Sep 17 00:00:00 2001
From: Tarwin Stroh-Spijer <tarwin@gmail.com>
Date: Mon, 5 Apr 2021 23:37:06 -0700
Subject: [PATCH 13/27] feat: very simple caching for feed calls

---
 index.js | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/index.js b/index.js
index d58ad9c..7ce1e8b 100644
--- a/index.js
+++ b/index.js
@@ -153,7 +153,7 @@ const randomFeed = async (counter, res) => {
     })
 }
 
-const getFeed = async (counter, res) => {
+const getFeed = async (counter) => {
 
     /*     const now_time = Date.now()
         const immutable = (typeof max_time !== 'undefined') && (max_time < now_time)
@@ -178,13 +178,13 @@ const getFeed = async (counter, res) => {
            cache_time = (int)(((max_time + ONE_MINUTE_MILLIS) - now_time) / 1000)
        } */
     var promise = Promise.all(feed.map(e => e))
-    promise.then(async (results) => {
+    return promise.then(async (results) => {
         var aux_arr = results.map(e => e)
 
         //res.set('Cache-Control', `public, max-age=${cache_time}`)
 
         console.log(aux_arr)
-        res.json({ result: aux_arr })
+        return aux_arr
     })
 }
 
@@ -292,16 +292,26 @@ const app = express()
 app.use(express.json())
 app.use(cors({ origin: '*' }))
 
+// used for very simple caching of the feed
+const feedCacheTimeLimit = ONE_MINUTE_MILLIS / 6 // 10 seconds
+const feedCache = {}
+
 app.post('/feed', async (req, res) => {
-    /*     
-        var counter = req.query.counter
-        var max_time = req.query.hasOwnProperty('time') ? customFloor(req.query.time, ONE_MINUTE_MILLIS) : null
-        const now_time_qt = customFloor(Date.now(), ONE_MINUTE_MILLIS)
-        if (max_time != null & max_time > now_time_qt) {
-            max_time = null
-        } 
-    */
-    await getFeed(req.body.counter, res)
+    const feedOffset = req.body.counter
+
+    if (feedCache[feedOffset] && Date.now() - feedCache[feedOffset].expires < feedCacheTimeLimit) {
+        return res.json({ result: feedCache[feedOffset].data })
+    }
+
+    // this should use a semaphore in case two requests come in at the same time
+    // for non-cached data
+    const aux_arr = await getFeed(feedOffset)
+    feedCache[feedOffset] = {
+        expires: Date.now(),
+        data: aux_arr
+    }
+
+    return res.json({ result: aux_arr })
 })
 
 app.post('/random', async (req, res) => {
@@ -337,9 +347,10 @@ app.post('/hdao', async (req, res) => {
     await hDAOFeed(parseInt(req.body.counter), res)
 })
 
-const testhdao = async () =>  await hDAOFeed(parseInt(0))
+// const testhdao = async () =>  await hDAOFeed(parseInt(0))
 //testhdao()
 
 app.listen(3001)
+console.log('Server running on localhost:3001')
 //module.exports.handler = serverless(app)
 

From 0134e0cc90b9a116d292f8e9081d15afdce0cdca Mon Sep 17 00:00:00 2001
From: Tarwin Stroh-Spijer <tarwin@gmail.com>
Date: Mon, 5 Apr 2021 23:51:56 -0700
Subject: [PATCH 14/27] feat: simple caching with semaphore locking

---
 index.js     | 7 +++++--
 package.json | 1 +
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index 7ce1e8b..2e27db3 100644
--- a/index.js
+++ b/index.js
@@ -6,6 +6,7 @@ const _ = require('lodash')
 const conseilUtil = require('./conseilUtil')
 const { random } = require('lodash')
 require('dotenv').config()
+const { Semaphore } =  require('prex')
 
 const reducer = (accumulator, currentValue) => parseInt(accumulator) + parseInt(currentValue)
 
@@ -295,21 +296,23 @@ app.use(cors({ origin: '*' }))
 // used for very simple caching of the feed
 const feedCacheTimeLimit = ONE_MINUTE_MILLIS / 6 // 10 seconds
 const feedCache = {}
+const feedLock = new Semaphore(1)
 
 app.post('/feed', async (req, res) => {
     const feedOffset = req.body.counter
 
+    await feedLock.wait()
     if (feedCache[feedOffset] && Date.now() - feedCache[feedOffset].expires < feedCacheTimeLimit) {
+        await feedLock.release()
         return res.json({ result: feedCache[feedOffset].data })
     }
 
-    // this should use a semaphore in case two requests come in at the same time
-    // for non-cached data
     const aux_arr = await getFeed(feedOffset)
     feedCache[feedOffset] = {
         expires: Date.now(),
         data: aux_arr
     }
+    await feedLock.release()
 
     return res.json({ result: aux_arr })
 })
diff --git a/package.json b/package.json
index aebde7f..c2880f5 100644
--- a/package.json
+++ b/package.json
@@ -25,6 +25,7 @@
     "lodash": "^4.17.21",
     "loglevel": "1.7.1",
     "node-fetch": "2.6.1",
+    "prex": "^0.4.7",
     "serverless-dotenv-plugin": "^3.8.1",
     "serverless-http": "^2.7.0"
   },

From 36c1e3eb6ffe07876866a22f7101500e589caf3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Tue, 6 Apr 2021 15:38:11 +0200
Subject: [PATCH 15/27] Make it more clear which is loading

---
 index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.js b/index.js
index cc2b1ee..3928057 100644
--- a/index.js
+++ b/index.js
@@ -162,7 +162,7 @@ const getFeed = async (counter, res, featured) => {
         const immutable = (typeof max_time !== 'undefined') && (max_time < now_time)
         max_time = (typeof max_time !== 'undefined') ? max_time : customFloor(now_time, ONE_MINUTE_MILLIS)
      */
-    console.log('feed')
+    console.log(`feed, featured: ${featured}`)
     var arr
     if (featured) {
         arr = await conseilUtil.getFeaturedArtisticUniverse(0)

From 10f3e30a1a530501e8ed71208251bd6b58ac5990 Mon Sep 17 00:00:00 2001
From: Tarwin Stroh-Spijer <tarwin@gmail.com>
Date: Tue, 6 Apr 2021 10:04:40 -0700
Subject: [PATCH 16/27] feat: better feed caching using offset feat: feed
 caching at 1 minute instead of 10 seconds as this is likely the update time
 of the blockchain

---
 index.js | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/index.js b/index.js
index 2e27db3..a17f34c 100644
--- a/index.js
+++ b/index.js
@@ -11,7 +11,6 @@ const { Semaphore } =  require('prex')
 const reducer = (accumulator, currentValue) => parseInt(accumulator) + parseInt(currentValue)
 
 const getIpfsHash = async (ipfsHash) => {
-
     return await axios.get('https://cloudflare-ipfs.com/ipfs/' + ipfsHash).then(res => res.data)
     /*    const nftDetailJson = await nftDetails.json();
    
@@ -47,14 +46,14 @@ const owners = async (obj) => {
     var values_arr = (_.values(owners))
     obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? values_arr.filter(e => parseInt(e) > 0).reduce(reducer) : 0
     obj.owners = owners
-    console.log(obj)
+    // console.log(obj)
     //obj.total_amount = (values_arr.map(e => parseInt(e))).reduce(reducer)
     return obj
 }
 
 const totalAmountIntegral = async (obj) => {
     var owners = await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + obj.token_id).then(res => res.data)
-    console.log(owners)
+    // console.log(owners)
     var values_arr = (_.values(owners))
     obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? (values_arr.filter(e => parseInt(e))) : 0
 
@@ -142,14 +141,14 @@ const randomFeed = async (counter, res) => {
     feed = await feed.map(async e => {
         e.token_info = await getIpfsHash(e.ipfsHash)
         e.token_id = parseInt(e.objectId)
-        console.log(e)
+        // console.log(e)
         return e
     })
     var promise = Promise.all(feed.map(e => e))
     promise.then(async (results) => {
         var aux_arr = results.map(e => e)
         //res.set('Cache-Control', `public, max-age=${cache_time}`)
-        console.log(aux_arr)
+        // console.log(aux_arr)
         res.json({ result: aux_arr })
     })
 }
@@ -163,7 +162,7 @@ const getFeed = async (counter) => {
     var arr = await conseilUtil.getArtisticUniverse(0)
 
     var feed = offset(desc(arr), counter)
-    console.log(feed)
+    // console.log(feed)
     feed = await feed.map(async e => {
         e.token_info = await getIpfsHash(e.ipfsHash)
         e.token_id = parseInt(e.objectId)
@@ -184,7 +183,7 @@ const getFeed = async (counter) => {
 
         //res.set('Cache-Control', `public, max-age=${cache_time}`)
 
-        console.log(aux_arr)
+        // console.log(aux_arr)
         return aux_arr
     })
 }
@@ -294,16 +293,24 @@ app.use(express.json())
 app.use(cors({ origin: '*' }))
 
 // used for very simple caching of the feed
-const feedCacheTimeLimit = ONE_MINUTE_MILLIS / 6 // 10 seconds
+const feedCacheTimeLimit = ONE_MINUTE_MILLIS // the blockchain updates about once a minute
 const feedCache = {}
-const feedLock = new Semaphore(1)
+const feedLocks = new Semaphore(1)
+
+const getFeedLock = function(offset) {
+    if (!feedLocks[offset]) {
+        feedLocks[offset] = new Semaphore(1)
+    }
+    return feedLocks[offset]
+}
 
 app.post('/feed', async (req, res) => {
-    const feedOffset = req.body.counter
+    const feedOffset = req.body.counter || 0
 
-    await feedLock.wait()
+    await getFeedLock(feedOffset).wait()
     if (feedCache[feedOffset] && Date.now() - feedCache[feedOffset].expires < feedCacheTimeLimit) {
-        await feedLock.release()
+        await getFeedLock(feedOffset).release()
+        // console.log('Feed from CACHE')
         return res.json({ result: feedCache[feedOffset].data })
     }
 
@@ -312,8 +319,8 @@ app.post('/feed', async (req, res) => {
         expires: Date.now(),
         data: aux_arr
     }
-    await feedLock.release()
-
+    await getFeedLock(feedOffset).release()
+    // console.log('Feed from NEW')
     return res.json({ result: aux_arr })
 })
 

From 8d7562753ea74f657b6929222adb40233184cff4 Mon Sep 17 00:00:00 2001
From: Tarwin Stroh-Spijer <tarwin@gmail.com>
Date: Tue, 6 Apr 2021 10:37:14 -0700
Subject: [PATCH 17/27] feat: restricted OBJKT and Address list cached for 1
 minute

---
 index.js | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 50 insertions(+), 6 deletions(-)

diff --git a/index.js b/index.js
index a17f34c..4cc9893 100644
--- a/index.js
+++ b/index.js
@@ -279,6 +279,50 @@ const hDAOFeed = async (counter, res) => {
     })
 }
 
+// list of restricted addresses
+const restrictedAdddressesCacheTimeLimit = ONE_MINUTE_MILLIS // the blockchain updates about once a minute
+let restrictedAddressesCache = null
+const restrictedAddressesLock = new Semaphore(1)
+const getRestrictedAddresses = async () => {
+    await restrictedAddressesLock.wait()
+    if (restrictedAddressesCache && Date.now() - restrictedAddressesCache.expires < restrictedAdddressesCacheTimeLimit) {
+        restrictedAddressesLock.release()
+        // console.log('ADDRESS restrictions from CACHE')
+        return restrictedAddressesCache.data
+    }
+
+    const list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/w.json').then(res => res.data)
+    restrictedAddressesCache = {
+        expires: Date.now(),
+        data: list
+    }
+    restrictedAddressesLock.release()
+    // console.log('ADDRESS restrictions from NEW')
+    return list
+}
+
+// list of restricted objkts
+const restrictedObjectsCacheTimeLimit = ONE_MINUTE_MILLIS // the blockchain updates about once a minute
+let restrictedObjectsCache = null
+const restrictedObjectsLock = new Semaphore(1)
+const getRestrictedObjkts = async () => {
+    await restrictedObjectsLock.wait()
+    if (restrictedObjectsCache && Date.now() - restrictedObjectsCache.expires < restrictedObjectsCacheTimeLimit) {
+        restrictedObjectsLock.release()
+        // console.log('OBJKT restrictions from CACHE')
+        return restrictedObjectsCache.data
+    }
+
+    const list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/o.json').then(res => res.data)
+    restrictedObjectsCache = {
+        expires: Date.now(),
+        data: list
+    }
+    restrictedObjectsLock.release()
+    // console.log('OBJKT restrictions from NEW')
+    return list
+}
+
 //getObjkts()
 //testSwaps()
 //getFeed(0)
@@ -295,9 +339,9 @@ app.use(cors({ origin: '*' }))
 // used for very simple caching of the feed
 const feedCacheTimeLimit = ONE_MINUTE_MILLIS // the blockchain updates about once a minute
 const feedCache = {}
-const feedLocks = new Semaphore(1)
+const feedLocks = {}
 
-const getFeedLock = function(offset) {
+const getFeedLock = (offset) => {
     if (!feedLocks[offset]) {
         feedLocks[offset] = new Semaphore(1)
     }
@@ -309,7 +353,7 @@ app.post('/feed', async (req, res) => {
 
     await getFeedLock(feedOffset).wait()
     if (feedCache[feedOffset] && Date.now() - feedCache[feedOffset].expires < feedCacheTimeLimit) {
-        await getFeedLock(feedOffset).release()
+        getFeedLock(feedOffset).release()
         // console.log('Feed from CACHE')
         return res.json({ result: feedCache[feedOffset].data })
     }
@@ -319,7 +363,7 @@ app.post('/feed', async (req, res) => {
         expires: Date.now(),
         data: aux_arr
     }
-    await getFeedLock(feedOffset).release()
+    getFeedLock(feedOffset).release()
     // console.log('Feed from NEW')
     return res.json({ result: aux_arr })
 })
@@ -331,7 +375,7 @@ app.post('/random', async (req, res) => {
 app.post('/tz', async (req, res) => {
 
     // list of restricted addresses
-    var list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/w.json').then(res => res.data)
+    var list = await getRestrictedAddresses()
 
     list.includes(req.body.tz)
         ?
@@ -344,7 +388,7 @@ app.post('/tz', async (req, res) => {
 app.post('/objkt', async (req, res) => {
 
     // list of restricted objkts
-    var list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/o.json').then(res => res.data)
+    var list = await getRestrictedObjkts()
 
     list.includes(parseInt(req.body.objkt_id))
         ?

From 75cbb61dc7b2b248871d3c748aac9f6d1b6fab75 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <42718855+anonymoussprocket@users.noreply.github.com>
Date: Tue, 6 Apr 2021 22:32:17 -0400
Subject: [PATCH 18/27] - increase query size

---
 conseilUtil.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index beb5845..a9c179f 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -245,7 +245,7 @@ const getArtisticUniverse = async (max_time) => {
     let royaltiesQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     royaltiesQuery = conseiljs.ConseilQueryBuilder.addFields(royaltiesQuery, 'key', 'value');
     royaltiesQuery = conseiljs.ConseilQueryBuilder.addPredicate(royaltiesQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftRoyaltiesMap])
-    royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 20_000)
+    royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 30_000)
     const royaltiesResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', royaltiesQuery)
     let artistMap = {}
     royaltiesResult.forEach(row => {
@@ -255,7 +255,7 @@ const getArtisticUniverse = async (max_time) => {
     let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value')
     swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap])
-    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 20_000)
+    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 30_000)
 
     const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery)
 

From 02d4e5766b5d2a051c85ddda0adacc92b5d93335 Mon Sep 17 00:00:00 2001
From: anonymoussprocket <42718855+anonymoussprocket@users.noreply.github.com>
Date: Tue, 6 Apr 2021 22:33:16 -0400
Subject: [PATCH 19/27] - resolving conflict

---
 conseilUtil.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index a9c179f..14cd732 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -255,7 +255,7 @@ const getArtisticUniverse = async (max_time) => {
     let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value')
     swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap])
-    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 30_000)
+    swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 30_000) // NOTE, limited to 30_000
 
     const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery)
 

From 4e6a1e930ad6aec525b244b4f1f2c243212c2316 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Wed, 7 Apr 2021 15:41:45 +0200
Subject: [PATCH 20/27] Let people's first mint through

---
 conseilUtil.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index a75f7b5..dd6faf1 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -460,7 +460,7 @@ const getFeaturedArtisticUniverse = async(max_time) => {
     thresholdHdao = Math.min(1_000_000, Math.max(100_000 * hdaoPerTez, 10_000))
 
     return artisticUniverse.filter(function (o) {
-        return ((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao
+        return (((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao) || (mintsPerCreator[o.minter] <= 1)
     })
 }
 

From 16bd0ba6369ce9427ec2f38054b962263090ce32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=B6ran=20Sandstr=C3=B6m?= <v3qt0r@gmail.com>
Date: Thu, 8 Apr 2021 08:14:30 +0200
Subject: [PATCH 21/27] Don't use local express in prod :D

---
 index.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index 7f75416..c13e29e 100644
--- a/index.js
+++ b/index.js
@@ -440,7 +440,7 @@ app.post('/hdao', async (req, res) => {
 // const testhdao = async () =>  await hDAOFeed(parseInt(0))
 //testhdao()
 
-app.listen(3001)
+//app.listen(3001)
 console.log('SERVER RUNNING ON localhost:3001')
-// module.exports.handler = serverless(app)
+module.exports.handler = serverless(app)
 

From 27ad4d3e0d1a2e052b79c311ba631e369a3796e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 8 Apr 2021 11:23:52 +0200
Subject: [PATCH 22/27] Make getArtisticUniverse quite a bit faster

---
 conseilUtil.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index bcade27..308578d 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -350,14 +350,17 @@ const getArtisticOutputForAddress = async (address) => {
 }
 
 const getArtisticUniverse = async (max_time) => {
+    var d = new Date();
+    d.setDate(d.getDate()-14);
     let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery();
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash');
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction'])
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [d.getTime()]) //Two weeks ago
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied'])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT'])
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 30_000)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 1000)
 
     const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData(
         { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },

From 207571f5778dcf3f1a1200417f0107a6706ed866 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 8 Apr 2021 11:25:04 +0200
Subject: [PATCH 23/27] Maybe 2500 is more reasonable

---
 conseilUtil.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index 308578d..bb962d6 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -360,7 +360,7 @@ const getArtisticUniverse = async (max_time) => {
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT'])
     mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC)
-    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 1000)
+    mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 2500)
 
     const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData(
         { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' },

From 7e1cfbf325a392c018c8b63c908a510fbbf6e928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 8 Apr 2021 11:49:45 +0200
Subject: [PATCH 24/27] Leniency ruined featured feed, don't listen to users

---
 conseilUtil.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index bb962d6..6a489e5 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -463,7 +463,7 @@ const getFeaturedArtisticUniverse = async(max_time) => {
     thresholdHdao = Math.min(1_000_000, Math.max(100_000 * hdaoPerTez, 10_000))
 
     return artisticUniverse.filter(function (o) {
-        return (((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao) || (mintsPerCreator[o.minter] <= 1)
+        return (((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao)
     })
 }
 

From 5f7e433d7f9a3a241aa4320409150186a397f5df Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 8 Apr 2021 12:00:23 +0200
Subject: [PATCH 25/27] Clean up lots of garbage

---
 conseilUtil.js |  11 +----
 index.js       | 132 -------------------------------------------------
 2 files changed, 2 insertions(+), 141 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index 6a489e5..96f267c 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -9,7 +9,6 @@ conseiljs.registerLogger(logger)
 conseiljs.registerFetch(fetch)
 const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech'
 const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud
-const tezosNode = ''
 const mainnet = require('./config').networkConfig
 
 
@@ -227,12 +226,10 @@ const gethDAOPerTez = async() => {
 
 const getKolibriPerTez = async() => {
     const tezBalance = await(getTezBalanceForAddress(mainnet.kolibriSwap))
-    //console.log("Tez balance", tezBalance)
     var kolibriBalance = await(getTokenBalance(mainnet.kolibriLedger, mainnet.kolibriSwap))
+
+    // TODO: Find a better way to get the balance, this is FA1.2, mike?
     kolibriBalance = parseInt(kolibriBalance.replace("Pair {} ", "")) / (10**((18 - 6)))
-    //console.log(kolibriBalance)
-    //console.log(typeof(kolibriBalance))
-    //console.log("Kol balance", kolibriBalance)
     return kolibriBalance / tezBalance
 }
 
@@ -470,11 +467,7 @@ const getFeaturedArtisticUniverse = async(max_time) => {
 const getRecommendedCurateDefault = async() => {
     hdaoPerTez = await gethDAOPerTez()
     kolPerTez = await getKolibriPerTez()
-    //console.log("hdaoPerTez", hdaoPerTez)
-    //console.log("kolPerTez", kolPerTez)
     hdaoPerKol = hdaoPerTez / kolPerTez
-    //console.log("hdaoPerKol", hdaoPerKol)
-
     //Minimum of $0.1, 0.1 hDAO, and 0.1tez, in hDAO
     return Math.floor(Math.min(hdaoPerKol * 0.1, 0.1, 0.1 * hdaoPerTez) * 1_000_000)
 }
diff --git a/index.js b/index.js
index c13e29e..f10c848 100644
--- a/index.js
+++ b/index.js
@@ -15,122 +15,20 @@ const reducer = (accumulator, currentValue) => parseInt(accumulator) + parseInt(
 
 const getIpfsHash = async (ipfsHash) => {
     return await axios.get('https://cloudflare-ipfs.com/ipfs/' + ipfsHash).then(res => res.data)
-    /*    const nftDetailJson = await nftDetails.json();
-   
-       const nftName = nftDetailJson.name;
-       const nftDescription = nftDetailJson.description;
-       const nftCreators = nftDetailJson.creators.join(', ');
-       const nftArtifact = `https://cloudflare-ipfs.com/ipfs/${nftDetailJson.formats[0].uri.toString().slice(7)}`;
-       const nftArtifactType = nftDetailJson.formats[0].mimeType.toString();
-   
-       return { name: nftName, description: nftDescription, creators: nftCreators, artifactUrl: nftArtifact, artifactType: nftArtifactType }; */
-}
-const getObjkts = async () => {
-    return await axios.get(`https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens`).then(res => res.data)
-}
-
-const getTokenHolders = async (tk_id) => {
-    return await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + tk_id).then(res => res.data)
 }
 
-const getTokenHoldersArr = async (arr) => {
-
-    return await arr.map(async e => await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + e).then(res => res.data))
-    /*     await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + arr[0]).then(res => console.log(res.data))
-     *//*     var result = arr.map(async e => {
-return await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + e).then(res => res.data)
-})
- 
-console.log(result) */
-}
 
 const owners = async (obj) => {
     var owners = await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + obj.token_id).then(res => res.data)
     var values_arr = (_.values(owners))
     obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? values_arr.filter(e => parseInt(e) > 0).reduce(reducer) : 0
     obj.owners = owners
-    // console.log(obj)
-    //obj.total_amount = (values_arr.map(e => parseInt(e))).reduce(reducer)
     return obj
 }
 
-const totalAmountIntegral = async (obj) => {
-    var owners = await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + obj.token_id).then(res => res.data)
-    // console.log(owners)
-    var values_arr = (_.values(owners))
-    obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? (values_arr.filter(e => parseInt(e))) : 0
-
-    obj.owners = owners
-    return obj
-}
-
-const objktAmount = async (arr) => {
-    return await arr.map(e => totalAmountIntegral(e))
-    //console.log(await getTokenHoldersArr(arr.map(e => _.values(e.token_id)[0])))
-}
-
-const objktOwners = async (arr) => {
-    return await arr.map(e => totalAmountIntegral(e))
-}
-
-
-const getObjktLedger = async () => await axios.get('https://better-call.dev/v1/bigmap/mainnet/511/keys?size=6500').then(res => res.data.map(e => ({ amount: parseInt(e.data.value.value), tz: e.data.key.children[0].value, tk_id: parseInt(e.data.key.children[1].value) })))
-const gethDAOLedger = async (counter) => await axios.get('https://api.better-call.dev/v1/bigmap/mainnet/519/keys?size=10&offset=' + counter * 10).then(res => res.data.map(e => {
-    return { token_id: parseInt(e.data.key.value), hDAO_balance: parseInt(e.data.value.children[0].value) }
-}))
-
-//gethDAOLedger()
-
-
-const getSwaps = async () => {
-    return await axios.get(`https://api.better-call.dev/v1/bigmap/mainnet/523/keys?size=6000`).then(res => {
-        return (res.data).map(e => {
-            var obj = {}
-
-            obj['swap_id'] = e.data.key.value
-            e.data.value != null ? e.data.value.children.map(e => obj[e.name] = e.value) : null
-            return obj
-        })
-    })
-}
-
-const merge = (a, b) => {
-    a.forEach((e1) => {
-        b.forEach((e2) => {
-            if (e1.token_id === e2.tk_id) {
-                _.assign(e1, e2)
-            }
-        })
-    })
-    return a
-}
-
-const mergeSwaps = (arr, swaps) => {
-    arr.forEach((e1) => {
-
-        e1.swaps = []
-
-        swaps.forEach((e2) => {
-            if (parseInt(e1.token_id) === parseInt(e2.objkt_id)) {
-                e1.swaps.push(e2)
-            }
-        })
-    })
-    return arr
-}
-
 const desc = arr => _.sortBy(arr, e => parseInt(e.objectId)).reverse()
 const offset = (arr, set) => arr.slice(set * 30, set * 30 + 30)
 
-const filter = (data, tz) => _.filter(data, (e) => {
-    if (e.token_info != undefined) {
-        return e.token_info.creators[0] === tz
-    }
-})
-
-const filterTz = (data, tz) => _.filter(data, { tz: tz })
-
-const test = async () => console.log(desc(await getObjkts()))
 
 const customFloor = function (value, roundTo) {
     return Math.floor(value / roundTo) * roundTo;
@@ -144,14 +42,11 @@ const randomFeed = async (counter, res) => {
     feed = await feed.map(async e => {
         e.token_info = await getIpfsHash(e.ipfsHash)
         e.token_id = parseInt(e.objectId)
-        // console.log(e)
         return e
     })
     var promise = Promise.all(feed.map(e => e))
     promise.then(async (results) => {
         var aux_arr = results.map(e => e)
-        //res.set('Cache-Control', `public, max-age=${cache_time}`)
-        // console.log(aux_arr)
         res.json({ result: aux_arr })
     })
 }
@@ -196,8 +91,6 @@ const getFeed = async (counter, featured) => {
     })
 }
 
-const filterObjkts = (arr, id_arr) => _.filter(arr, { token_id: tk.id })
-
 const getTzLedger = async (tz, res) => {
     /*     var ledger = desc(await getObjktLedger())
         var objkts = await getObjkts()
@@ -258,8 +151,6 @@ const getTzLedger = async (tz, res) => {
             hdao: hdao
         })
     })
-
-    //return tzLedger
 }
 
 const getObjktById = async (id, res) => {
@@ -270,20 +161,6 @@ const getObjktById = async (id, res) => {
     console.log(objkt)
 
     return objkt
-    //res.json({ result : objkt })
-    //var objkts = await getObjkts()
-    //var swaps = await getSwaps()
-    //res.json({ result : mergeSwaps([objkt], swaps)[0] })
-    //console.log(_.filter(mergeSwaps(objkts, swaps), {token_id : id}))
-    //   var arr = await objktOwners(_.filter(mergeSwaps(objkts, swaps), {token_id : id}))
-    //   var promise = Promise.all(arr.map(e => e))
-
-    /*     promise.then((results) => {
-            var aux_arr = results.map(e => e)
-            console.log(aux_arr)
-            res.json({ result : aux_arr })
-        }) */
-    //https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=842
 }
 
 const mergehDAO = async (obj) => {
@@ -348,18 +225,9 @@ const getRestrictedObjkts = async () => {
         data: list
     }
     restrictedObjectsLock.release()
-    // console.log('OBJKT restrictions from NEW')
     return list
 }
 
-//getObjkts()
-//testSwaps()
-//getFeed(1)
-//getTzLedger('tz1UBZUkXpKGhYsP5KtzDNqLLchwF4uHrGjw')
-//getObjktById(15306)
-//const test2 = async () => console.log(await getObjktLedger())
-//test2()
-
 const app = express()
 
 app.use(express.json())

From 46f1274958557d436ec6a831ceddb8ab3cabb50c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Thu, 8 Apr 2021 15:36:27 +0200
Subject: [PATCH 26/27] Speed up curate using 5 min cache

---
 index.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/index.js b/index.js
index f10c848..a986940 100644
--- a/index.js
+++ b/index.js
@@ -298,6 +298,7 @@ app.post('/objkt', async (req, res) => {
 
 app.get('/recommend_curate', async (req, res) => {
     const amt = await conseilUtil.getRecommendedCurateDefault()
+    res.set('Cache-Control', `public, max-age=300`)
     res.json({ amount: amt })
 })
 

From 4d75c64c546251f2d1f005c97153485817514d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Go=CC=88ran=20Sandstro=CC=88m?=
 <goran.sandstrom@bookbeat.com>
Date: Fri, 9 Apr 2021 13:22:26 +0200
Subject: [PATCH 27/27] Fix owners returning 0 for recently minted objekts

---
 conseilUtil.js | 30 +++++++++++++++++++++++++++++-
 index.js       |  2 +-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/conseilUtil.js b/conseilUtil.js
index 96f267c..ef1e4cf 100644
--- a/conseilUtil.js
+++ b/conseilUtil.js
@@ -261,6 +261,33 @@ const gethDaoBalances = async () => {
 
 }
 
+const getObjektOwners = async (objekt_id) => {
+    let objektBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery();
+    objektBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(objektBalanceQuery, 'key', 'value');
+    objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger])
+    objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'key', conseiljs.ConseilOperator.ENDSWITH, [` ${objekt_id}`], false)
+    objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true)
+    objektBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(objektBalanceQuery, 500_000)
+
+    let objektMap = {}
+
+    try {
+        const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objektBalanceQuery);
+        
+
+        balanceResult.forEach(row => {
+            objektMap[conseiljs.TezosMessageUtils.readAddress(row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1'))] = row['value']
+        })
+        //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it
+    } catch (error) {
+        console.log(`getObjektOwners failed for ${JSON.stringify(objektBalanceQuery)} with ${error}`)
+    }
+
+
+    return objektMap
+
+}
+
 const getObjektMintingsLastWeek = async () => {
     var d = new Date();
     d.setDate(d.getDate()-5);
@@ -538,5 +565,6 @@ module.exports = {
     getArtisticUniverse,
     getFeaturedArtisticUniverse,
     hDAOFeed,
-    getRecommendedCurateDefault
+    getRecommendedCurateDefault,
+    getObjektOwners
 }
diff --git a/index.js b/index.js
index a986940..5c68d2a 100644
--- a/index.js
+++ b/index.js
@@ -19,7 +19,7 @@ const getIpfsHash = async (ipfsHash) => {
 
 
 const owners = async (obj) => {
-    var owners = await axios.get('https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + obj.token_id).then(res => res.data)
+    var owners = await conseilUtil.getObjektOwners(obj.token_id)
     var values_arr = (_.values(owners))
     obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? values_arr.filter(e => parseInt(e) > 0).reduce(reducer) : 0
     obj.owners = owners