remove grams from main dependencies (#15)

The GRAMS package causes lots of trouble installing SAND. Moreover, its continuously development makes it harder to have a stable SAND. This PR removes GRAMS from the main dependencies. Future work on using GRAMS as assistant will need to add it as optional dependency.

.gitignore

@@ -138,4 +138,5 @@ dmypy.json
 sand.db
 poetry.lock
 
-sand/www
+sand/www
+/data

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "web-sand"
-version = "2.1.13"
+version = "2.1.14"
 description = "UI for browsing/editing semantic descriptions"
 authors = ["Binh Vu <[email protected]>"]
 repository = "https://github.com/usc-isi-i2/sand"
@@ -15,9 +15,8 @@ include = ["sand/www/**/*"]
 sand = 'sand.__main__:cli'
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.11"
-kgdata = "^3.4.2"
-sm-grams = "^2.1.7"
+python = ">=3.8"
+kgdata = "^3.8.0"
 sem-desc = "^4.4.2"
 peewee = "^3.15.2"
 Flask = "^2.2.2"

sand/client.py

@@ -1,16 +1,17 @@
 from dataclasses import asdict
+from functools import lru_cache
+from pathlib import Path
 from typing import Union
+
+import requests
+import serde.json
 from gena.client import Client as IndividualClient
-from grams.inputs.linked_table import LinkedTable
-from pathlib import Path
 from peewee import fn
-import requests
-from sand.models import init_db, SemanticModel, Project, Table
-from sand.models.table import CandidateEntity, Link
 from sm.prelude import M
 from tqdm import tqdm
-import serde.json
-from functools import lru_cache
+
+from sand.models import Project, SemanticModel, Table, init_db
+from sand.models.table import CandidateEntity, Link
 
 
 class Client:

sand/commands/load.py

@@ -1,16 +1,21 @@
 from pathlib import Path
+from typing import List, Tuple
 
 import click
+from sm.dataset import Dataset, Example, FullTable
 from tqdm.auto import tqdm
+
 from sand.models import (
+    ContextPage,
+    Link,
     Project,
     SemanticModel,
-    db as dbconn,
-    init_db,
+    Table,
+    TableRow,
+    Value,
 )
-from sand.extensions.assistants.grams_plugin import convert_linked_table
-from sm.dataset import Example, Dataset
-from grams.inputs import LinkedTable
+from sand.models import db as dbconn
+from sand.models import init_db
 
 
 @click.command(name="load")
@@ -33,22 +38,13 @@ def load_dataset(db: str, project: str, dataset: str, n_tables: int):
     if n_tables > 0:
         examples = examples[:n_tables]
 
-    newexamples: list[Example[LinkedTable]] = []
-    for example in examples:
-        newexamples.append(
-            Example(
-                sms=example.sms,
-                table=LinkedTable.from_full_table(example.table),
-            )
-        )
-
     with dbconn:
         p = Project.get(name=project)
-        for e in tqdm(newexamples, desc="Loading examples"):
+        for e in tqdm(examples, desc="Loading examples"):
             save_example(p, e)
 
 
-def save_example(project: Project, example: Example[LinkedTable]):
+def save_example(project: Project, example: Example[FullTable]):
     mtbl, mrows = convert_linked_table(example.table)
     mtbl.project = project
     mtbl.save()
@@ -64,3 +60,61 @@ def save_example(project: Project, example: Example[LinkedTable]):
             version=1,
             data=sm,
         ).save()
+
+
+def convert_linked_table(tbl: FullTable) -> Tuple[Table, List[TableRow]]:
+    """Convert linked table to smc's tables and rows.
+    Note that the project of the table is not set, which is supposed to be set later.
+    """
+    context_values = [
+        Value("entityid", str(entityid)) for entityid in tbl.context.page_entities
+    ]
+
+    if tbl.context.page_url is not None:
+        context_page = ContextPage(
+            tbl.context.page_url,
+            tbl.context.page_title or "",
+            str(tbl.context.page_entities[0])
+            if len(tbl.context.page_entities) > 0
+            else None,
+        )
+    else:
+        context_page = None
+
+    nrows, ncols = tbl.table.shape()
+    links = {}
+    for ri in range(nrows):
+        links[ri] = {}
+
+    for ri, ci, lst in tbl.links.enumerate_flat_iter():
+        if len(lst) == 0:
+            continue
+        links[ri][ci] = [
+            Link(
+                start=l.start,
+                end=l.end,
+                url=l.url,
+                entity_id=str(l.entities[0]) if len(l.entities) > 0 else None,
+                candidate_entities=[],  # do not load the entities because this is filled by methods
+            )
+            for l in lst
+        ]
+
+    columns = [col.name for col in tbl.table.columns]
+    mtbl = Table(
+        name=tbl.table.table_id,
+        description="",
+        columns=columns,
+        size=nrows,
+        context_values=context_values,
+        context_tree=[],
+        context_page=context_page,
+        project=None,
+    )
+    mrows = []
+
+    for ri in range(nrows):
+        row = [tbl.table[ri, ci] for ci in range(len(columns))]
+        mrows.append(TableRow(table=mtbl, index=ri, row=row, links=links[ri]))
+
+    return mtbl, mrows

sand/controllers/assistant.py

@@ -1,29 +1,22 @@
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
 import threading
-from typing import Dict, List, Optional, Tuple
-from flask.blueprints import Blueprint
-from gena.deserializer import (
-    generate_deserializer,
-    get_dataclass_deserializer,
-)
-from grams.algorithm.inferences.features.tree_utils import TreeStruct
-from sm.misc.funcs import import_func
-from sm.outputs.semantic_model import SemanticModel
-from sand.config import SETTINGS
+from dataclasses import dataclass
 from functools import partial
-from sand.models.table import Link, Table, TableRow
-from flask import request, jsonify
-from sand.serializer import batch_serialize_sms, get_label, serialize_graph
-from sand.models.entity import NIL_ENTITY, Entity, EntityAR
-from sand.models.ontology import OntProperty, OntClass, OntPropertyAR, OntClassAR
-from sand.extension_interface.assistant import IAssistant
-from operator import attrgetter
-
-from peewee import Model as PeeweeModel, DoesNotExist, fn
+from typing import Dict, List, Optional
 
+from flask import jsonify, request
+from flask.blueprints import Blueprint
+from gena.deserializer import get_dataclass_deserializer
+from peewee import DoesNotExist
+from sm.misc.funcs import import_func
 from werkzeug.exceptions import BadRequest, NotFound
 
+from sand.config import SETTINGS
+from sand.extension_interface.assistant import IAssistant
+from sand.helpers.tree_utils import TreeStruct
+from sand.models.entity import NIL_ENTITY, Entity, EntityAR
+from sand.models.ontology import OntClass, OntClassAR, OntPropertyAR
+from sand.models.table import Link, Table, TableRow
+from sand.serializer import get_label, serialize_graph
 
 assistant_bp = Blueprint("assistant", "assistant")
 
@@ -45,7 +38,6 @@ def get_assistants() -> Dict[str, IAssistant]:
 
 @assistant_bp.route(f"/{assistant_bp.name}/predict/<table_id>", methods=["GET"])
 def predict_semantic_desc(table_id: int):
-
     table = Table.get_by_id(table_id)
     rows: List[TableRow] = list(
         TableRow.select().where(TableRow.table == table).order_by(TableRow.index)