Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hub changes for stream/repost with extensions #113

Open
wants to merge 13 commits into
base: master
Choose a base branch
from
Open
Prev Previous commit
Next Next commit
Indexing of extensions and basic query support.
moodyjon committed Nov 21, 2022
commit 1991513e158df4a678e8547885a8c4857f0b0df0
34 changes: 31 additions & 3 deletions hub/common.py
Original file line number Diff line number Diff line change
@@ -764,7 +764,8 @@ def __init__(self, got_version, expected_version):
"claim_type": {"type": "byte"},
"censor_type": {"type": "byte"},
"trending_score": {"type": "double"},
"release_time": {"type": "long"}
"release_time": {"type": "long"},
"extensions": {"type": "object"},
}
}
}
@@ -788,7 +789,7 @@ def __init__(self, got_version, expected_version):
'reposted_claim_id', 'repost_count', 'sd_hash',
'trending_score', 'tx_num',
'channel_tx_id', 'channel_tx_position', 'channel_height', 'reposted_tx_id',
'reposted_tx_position', 'reposted_height',
'reposted_tx_position', 'reposted_height', 'extensions',
}

TEXT_FIELDS = {
@@ -807,7 +808,11 @@ def __init__(self, got_version, expected_version):
'channel_tx_position', 'channel_height',
}

ALL_FIELDS = RANGE_FIELDS | TEXT_FIELDS | FIELDS
OBJECT_FIELDS = {
'extensions',
}

ALL_FIELDS = OBJECT_FIELDS | RANGE_FIELDS | TEXT_FIELDS | FIELDS

REPLACEMENTS = {
'claim_name': 'normalized_name',
@@ -826,6 +831,7 @@ def __init__(self, got_version, expected_version):


def expand_query(**kwargs):
#print(f'expand_query: >>> {kwargs}')
if "amount_order" in kwargs:
kwargs["limit"] = 1
kwargs["order_by"] = "effective_amount"
@@ -849,6 +855,7 @@ def expand_query(**kwargs):
if value is None or isinstance(value, list) and len(value) == 0:
continue
key = REPLACEMENTS.get(key, key)
#print(f'expand_query: *** {key} = {value}')
if key in FIELDS:
partial_id = False
if key == 'claim_type':
@@ -911,6 +918,26 @@ def expand_query(**kwargs):
]}
}
)
elif key in OBJECT_FIELDS:
def flatten(field, d):
if isinstance(d, dict) and len(d) > 0:
for k, v in d.items():
subfield = f'{field}.{k}' if field else k
yield from flatten(subfield, v)
elif isinstance(d, dict):
# require <field> be present
yield {"exists": {"field": field}}
elif isinstance(d, list):
# require <field> match all values <d>
yield {"bool": {"must": [{"match": {field: {"query": e}}} for e in d]}}
else:
# require <field> match value <d>
yield {"match": {field: {"query": d}}}
#yield {"term": {field: {"value": d}}}
query['must'].append(
{"exists": {"field": key}},
)
query['must'].extend(flatten(key, value))
elif many:
query['must'].append({"terms": {key: value}})
else:
@@ -1022,6 +1049,7 @@ def expand_query(**kwargs):
"sort": query["sort"]
}
}
#print(f'expand_query: <<< {query}')
return query


13 changes: 12 additions & 1 deletion hub/elastic_sync/db.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from typing import Optional, Set, Dict, List
from concurrent.futures.thread import ThreadPoolExecutor
from hub.schema.claim import guess_stream_type
from hub.schema.claim import guess_stream_type, Claim
from hub.schema.result import Censor
from hub.common import hash160, STREAM_TYPES, CLAIM_TYPES, LRUCache
from hub.db import SecondaryDB
@@ -43,9 +43,11 @@ async def prepare_claim_metadata_batch(self, claims: Dict[bytes, ResolveResult],
metadatas.update(await self.get_claim_metadatas(list(needed_txos)))

for claim_hash, claim in claims.items():
assert isinstance(claim, ResolveResult)
metadata = metadatas.get((claim.tx_hash, claim.position))
if not metadata:
continue
assert isinstance(metadata, Claim)
if not metadata.is_stream or not metadata.stream.has_fee:
fee_amount = 0
else:
@@ -98,16 +100,24 @@ async def prepare_claim_metadata_batch(self, claims: Dict[bytes, ResolveResult],
if reposted_metadata.is_stream and \
(reposted_metadata.stream.video.duration or reposted_metadata.stream.audio.duration):
reposted_duration = reposted_metadata.stream.video.duration or reposted_metadata.stream.audio.duration

extensions = None
if metadata.is_stream:
meta = metadata.stream
extensions = meta.extensions.to_dict()
elif metadata.is_channel:
meta = metadata.channel
elif metadata.is_collection:
meta = metadata.collection
elif metadata.is_repost:
meta = metadata.repost
modified = meta.reference.apply(reposted_metadata)
modified = getattr(modified, modified.claim_type)
if hasattr(modified, 'extensions'):
extensions = modified.extensions.to_dict()
else:
continue

claim_tags = [tag for tag in meta.tags]
claim_languages = [lang.language or 'none' for lang in meta.languages] or ['none']
tags = list(set(claim_tags).union(set(reposted_tags)))
@@ -181,6 +191,7 @@ async def prepare_claim_metadata_batch(self, claims: Dict[bytes, ResolveResult],
'channel_tx_id': None if not claim.channel_tx_hash else claim.channel_tx_hash[::-1].hex(),
'channel_tx_position': claim.channel_tx_position,
'channel_height': claim.channel_height,
'extensions': extensions
}

if metadata.is_repost and reposted_duration is not None:
12 changes: 7 additions & 5 deletions hub/elastic_sync/service.py
Original file line number Diff line number Diff line change
@@ -134,6 +134,8 @@ async def start_index(self) -> bool:
index_version = await self.get_index_version()

res = await self.sync_client.indices.create(self.index, INDEX_DEFAULT_SETTINGS, ignore=400)
if 'error' in res:
self.log.warning("es index create failed: %s", res)
acked = res.get('acknowledged', False)

if acked:
@@ -202,13 +204,13 @@ async def batched_update_filter(items: typing.Dict[bytes, bytes], channel: bool,

@staticmethod
def _upsert_claim_query(index, claim):
return {
'doc': {key: value for key, value in claim.items() if key in ALL_FIELDS},
doc = {key: value for key, value in claim.items() if key in ALL_FIELDS}
doc.update({
'_id': claim['claim_id'],
'_index': index,
'_op_type': 'update',
'doc_as_upsert': True
}
'_op_type': 'index',
})
return doc
moodyjon marked this conversation as resolved.
Show resolved Hide resolved

@staticmethod
def _delete_claim_query(index, claim_hash: bytes):
Loading