Cleanup

dhlab/api/dhlab_api.py

@@ -503,7 +503,7 @@ def ngram_news(
     df.columns = columns
     return df
 
-def create_sparse_matrix(structure):
+def _create_sparse_matrix(structure):
     """Create a sparse matrix from an API counts object"""
 
     # fetch all words
@@ -557,7 +557,7 @@ def get_document_frequencies(
                 pass
 
         if sparse == True:
-            df = create_sparse_matrix(structure)
+            df = _create_sparse_matrix(structure)
         else:
             df = pd.DataFrame(structure)
 
@@ -611,10 +611,6 @@ def get_urn_frequencies(urns: List[str] = None, dhlabid: List = None) -> DataFra
     return df
 
 
-def get_document_corpus(**kwargs):
-    return document_corpus(**kwargs)
-
-
 def document_corpus(
     doctype: str = None,
     author: str = None,
@@ -678,6 +674,9 @@ def document_corpus(
     return pd.DataFrame(r.json())
 
 
+get_document_corpus = document_corpus # Function alias
+
+
 def urn_collocation(
     urns: List = None,
     word: str = "arbeid",

dhlab/legacy/__init__.py

@@ -1 +1 @@
-
+

dhlab/text/corpus.py

@@ -223,11 +223,10 @@ def evaluate_words(self, wordbags=None):
     def add(self, new_corpus: Union[DataFrame, type("Corpus")]):
         """Utility for appending Corpus or DataFrame to self"""
         if isinstance(new_corpus, Corpus):
-            new_corpus = new_corpus.frame
+            new_corpus = new_corpus.corpus
         self.frame = pd.concat([self.frame, new_corpus])
         self.corpus = self.frame
         self._drop_urn_duplicates()
-        # self.size = len(self.frame)
 
     def sample(self, n: int = 5):
         """Create random subkorpus with `n` entries"""
@@ -345,8 +344,11 @@ def check_integrity(self):
 
         def test_dhlabid_series(series: pd.Series) -> bool:
             """Check if dhlabid series is valid"""
-            if not series.apply(lambda x: isinstance(x, int)).all():
+            try:
+                series = series.apply(lambda x: int(x))
+            except ValueError:
                 return False
+
             if not ((series >= 1e8) & (series < 1e9)).all():
                 return False
 
@@ -356,11 +358,7 @@ def test_urn_series(series: pd.Series) -> bool:
             """Check if URN series is valid"""
             if series.str.startswith("URN:NBN:no-nb_").all():
                 return True
-            try:
-                series = series.apply(lambda x: int(x))
-                return test_dhlabid_series(series)
-            except:
-                return False
+            return False
 
         # Check if the DataFrame is empty
         if self.corpus.empty:
@@ -377,7 +375,8 @@ def test_urn_series(series: pd.Series) -> bool:
             raise ValueError("Some dhlabid values are in an incorrect format.")
 
         # Validate URN format
-        if not test_urn_series(self.corpus["urn"]):
+        if not (test_urn_series(self.corpus["urn"])
+                or test_dhlabid_series(self.corpus["urn"])):
             raise ValueError("Some URN values are in an incorrect format.")
 
         return True

dhlab/text/parse.py

@@ -1,5 +1,3 @@
-import pandas as pd
-
 from dhlab.api.dhlab_api import ner_from_urn, pos_from_urn, show_spacy_models
 
 

dhlab/visualize/cloud.py

@@ -1,69 +1,69 @@
-import json
-from collections import Counter
-
-import matplotlib.pyplot as plt
-
-try:
-    from wordcloud import WordCloud
-except BaseException:
-    print("wordcloud er ikke installert, kan ikke lage ordskyer")
-
-
-def make_cloud(
-    json_text,
-    top=100,
-    background="white",
-    stretch=lambda x: 2 ** (10 * x),
-    width=500,
-    height=500,
-    font_path=None,
-):
-    """Create a word cloud from a frequency list."""
-    pairs0 = Counter(json_text).most_common(top)
-    pairs = {x[0]: stretch(x[1]) for x in pairs0}
-    wc = WordCloud(
-        font_path=font_path,
-        background_color=background,
-        width=width,
-        # color_func=my_colorfunc,
-        ranks_only=True,
-        height=height,
-    ).generate_from_frequencies(pairs)
-    return wc
-
-
-def draw_cloud(sky, width=20, height=20, fil=""):
-    """Draw a word cloud produced by :func:`make_cloud`."""
-    plt.figure(figsize=(width, height))
-    plt.imshow(sky, interpolation="bilinear")
-    figplot = plt.gcf()
-    if fil != "":
-        figplot.savefig(fil, format="png")
-
-
-def cloud(
-    df,
-    column="",
-    top=200,
-    width=1000,
-    height=1000,
-    background="black",
-    file="",
-    stretch=10,
-    font_path=None,
-):
-    """Make and draw a cloud from a pandas dataframe, using :func:`make_cloud` and
-    :func:`draw_cloud`."""
-    if column == "":
-        column = df.columns[0]
-    data = json.loads(df[column].to_json())
-    a_cloud = make_cloud(
-        data,
-        top=top,
-        background=background,
-        font_path=font_path,
-        stretch=lambda x: 2 ** (stretch * x),
-        width=width,
-        height=height,
-    )
-    draw_cloud(a_cloud, fil=file)
+import json
+from collections import Counter
+
+import matplotlib.pyplot as plt
+
+try:
+    from wordcloud import WordCloud
+except BaseException:
+    print("wordcloud er ikke installert, kan ikke lage ordskyer")
+
+
+def make_cloud(
+    json_text,
+    top=100,
+    background="white",
+    stretch=lambda x: 2 ** (10 * x),
+    width=500,
+    height=500,
+    font_path=None,
+):
+    """Create a word cloud from a frequency list."""
+    pairs0 = Counter(json_text).most_common(top)
+    pairs = {x[0]: stretch(x[1]) for x in pairs0}
+    wc = WordCloud(
+        font_path=font_path,
+        background_color=background,
+        width=width,
+        # color_func=my_colorfunc,
+        ranks_only=True,
+        height=height,
+    ).generate_from_frequencies(pairs)
+    return wc
+
+
+def draw_cloud(sky, width=20, height=20, fil=""):
+    """Draw a word cloud produced by :func:`make_cloud`."""
+    plt.figure(figsize=(width, height))
+    plt.imshow(sky, interpolation="bilinear")
+    figplot = plt.gcf()
+    if fil != "":
+        figplot.savefig(fil, format="png")
+
+
+def cloud(
+    df,
+    column="",
+    top=200,
+    width=1000,
+    height=1000,
+    background="black",
+    file="",
+    stretch=10,
+    font_path=None,
+):
+    """Make and draw a cloud from a pandas dataframe, using :func:`make_cloud` and
+    :func:`draw_cloud`."""
+    if column == "":
+        column = df.columns[0]
+    data = json.loads(df[column].to_json())
+    a_cloud = make_cloud(
+        data,
+        top=top,
+        background=background,
+        font_path=font_path,
+        stretch=lambda x: 2 ** (stretch * x),
+        width=width,
+        height=height,
+    )
+    draw_cloud(a_cloud, fil=file)