Skip to content

Commit

Permalink
Merge pull request #140 from usc-isi-i2/dev
Browse files Browse the repository at this point in the history 
merging dev to master for a release
  • Loading branch information
@saggu
saggu authored Sep 10, 2020
2 parents 307a1c6 + 15d50a2 commit e8b45e3
Show file tree
Hide file tree
Showing 18 changed files with 700 additions and 172 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ install:
script:
- python -m spacy download en_core_web_sm
- cd kgtk/tests
- coverage run --omit */site-packages/* -m unittest discover
- coverage run --source=kgtk -m unittest discover
after_success:
- coveralls
notifications:
Expand Down
6 changes: 6 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ An example command that computes instances of the subclasses of two classes:
kgtk instances --transitive --class Q13442814,Q12345678
```

## Running unit tests locally
```
cd kgtk/tests
python -W ignore -m unittest discover
```

## How to cite

```
Expand Down
2 changes: 1 addition & 1 deletion kgtk/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.3.2'
__version__ = '0.4.0'
2 changes: 1 addition & 1 deletion kgtk/cli/import_framenet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def edge2KGTK(edge: Tuple[str, str, str]) -> pd.Series:

def clean(e: str) -> str:
out = e.split(':')[-1].replace('_', ' ')
return KgtkFormat.stringify(re.sub("([a-z])([A-Z])", "\g<1> \g<2>", out).strip())
return KgtkFormat.stringify(re.sub("([a-z])([A-Z])", "\g<1> \g<2>", out).strip().lower())

return pd.Series({
'node1': s,
Expand Down
26 changes: 18 additions & 8 deletions kgtk/cli/import_wordnet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@


import sys
from kgtk.cli_argparse import KGTKArgumentParser, KGTKFiles

def parser():
return {
Expand All @@ -20,8 +21,9 @@ def add_arguments(parser):
Args:
parser (argparse.ArgumentParser)
"""
parser.add_output_file()

def run():
def run(output_file: KGTKFiles):

# import modules locally
import sys # type: ignore
Expand All @@ -30,10 +32,7 @@ def run():
import nltk
from nltk.corpus import wordnet as wn
from kgtk.kgtkformat import KgtkFormat

def header_to_edge(row):
row=[r.replace('_', ';') for r in row]
return '\t'.join(row) + '\n'
from kgtk.io.kgtkwriter import KgtkWriter

def obtain_wordnet_lemmas(syn):
lemmas=[]
Expand Down Expand Up @@ -109,8 +108,19 @@ def create_edges(data, labels, rel, rel_label):
return all_rows

try:
out_columns=['node1', 'relation', 'node2', 'node1_label', 'node2_label','relation_label', 'relation_dimension', 'source', 'sentence']
sys.stdout.write(header_to_edge(out_columns))
out_columns=['node1', 'relation', 'node2', 'node1;label', 'node2;label','relation;label', 'relation;dimension', 'source', 'sentence']

output_kgtk_file: Path = KGTKArgumentParser.get_output_file(output_file)
ew: KgtkWriter = KgtkWriter.open(out_columns,
output_kgtk_file,
#mode=input_kr.mode,
require_all_columns=False,
prohibit_extra_columns=True,
fill_missing_columns=True,
gzip_in_parallel=False,
#verbose=self.verbose,
#very_verbose=self.very_verbose
)

all_labels, all_hyps, all_members, all_parts, all_subs=get_wn_data()
hyp_edges=create_edges(all_hyps, all_labels, '/r/IsA', KgtkFormat.stringify('is a'))
Expand All @@ -120,7 +130,7 @@ def create_edges(data, labels, rel, rel_label):
all_edges=hyp_edges+member_edges+part_edges+sub_edges

for edge in all_edges:
sys.stdout.write('\t'.join(edge) + '\n')
ew.write(edge)

except Exception as e:
kgtk_exception_auto_handler(e)
35 changes: 0 additions & 35 deletions kgtk/cli/merge_identical_nodes.py
View file

This file was deleted.

4 changes: 2 additions & 2 deletions kgtk/iff/kgtkifexists.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,8 @@ def process_cacheing_filter(self,
if self.verbose:
print("Filtering records from %s" % self.input_file_path, file=self.error_file, flush=True)
input_line_count: int = 0
accept_line_count: int = 0;
reject_line_count: int = 0;
accept_line_count: int = 0
reject_line_count: int = 0

# TODO: join these two code paths using xor?
row: typing.List[str]
Expand Down
2 changes: 2 additions & 0 deletions kgtk/tests/data/Q47158_non_edge.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
heading
Q47158
119 changes: 1 addition & 118 deletions kgtk/tests/data/sample_kgtk_edge_Q47158.tsv
View file
Original file line number Diff line number Diff line change
@@ -1,119 +1,2 @@
id node1 label node2 rank
Q47158-P1014-1 Q47158 P1014 300391257 normal
Q47158-P1051-1 Q47158 P1051 4621 normal
Q47158-P1296-1 Q47158 P1296 0239797 normal
Q47158-P1343-1 Q47158 P1343 Q20078554 normal
Q47158-P1417-1 Q47158 P1417 science/Triassic-Period normal
Q47158-P155-1 Q47158 P155 Q76402 normal
Q47158-P156-1 Q47158 P156 Q45805 normal
Q47158-P18-1 Q47158 P18 Sketch plateosaurus bulky.jpg normal
Q47158-P227-1 Q47158 P227 4186023-8 normal
Q47158-P2924-1 Q47158 P2924 4201699 normal
Q47158-P31-1 Q47158 P31 Q392928 normal
Q47158-P31-2 Q47158 P31 Q403523 normal
Q47158-P3219-1 Q47158 P3219 trias normal
Q47158-P349-1 Q47158 P349 00570199 normal
Q47158-P361-1 Q47158 P361 Q83222 normal
Q47158-P373-1 Q47158 P373 Triassic normal
Q47158-P4210-1 Q47158 P4210 13350 normal
Q47158-P4306-1 Q47158 P4306 1260 normal
Q47158-P508-1 Q47158 P508 32513 normal
Q47158-P527-1 Q47158 P527 Q917293 normal
Q47158-P527-2 Q47158 P527 Q929395 normal
Q47158-P527-3 Q47158 P527 Q1075885 normal
Q47158-P580-1 Q47158 P580 ^-252170000-00-00T00:00:00Z/5 normal
Q47158-P582-1 Q47158 P582 ^-201300000-00-00T00:00:00Z/4 normal
Q47158-P646-1 Q47158 P646 /m/07csq normal
Q47158-P7033-1 Q47158 P7033 scot/8352 normal
Q47158-P7818-1 Q47158 P7818 Trias normal
Q47158-P7832-1 Q47158 P7832 Triasiko normal
Q47158-P8408-1 Q47158 P8408 Triassic-GeologicalPeriod normal
Q47158-P910-1 Q47158 P910 Q6431022 normal
Q47158-description-1 Q47158 description first period of the Mesozoic Era
Q47158-label-1 Q47158 label Triassic
Q47158-wikipedia_sitelink-1 Q47158 wikipedia_sitelink http://lb.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-10 Q47158 wikipedia_sitelink http://br.wikipedia.org/wiki/Triaseg
Q47158-wikipedia_sitelink-11 Q47158 wikipedia_sitelink http://el.wikipedia.org/wiki/Τριαδική_περίοδος
Q47158-wikipedia_sitelink-12 Q47158 wikipedia_sitelink http://ga.wikipedia.org/wiki/Triasach
Q47158-wikipedia_sitelink-13 Q47158 wikipedia_sitelink http://pnb.wikipedia.org/wiki/ٹرائیسک
Q47158-wikipedia_sitelink-14 Q47158 wikipedia_sitelink http://nl.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-15 Q47158 wikipedia_sitelink http://sv.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-16 Q47158 wikipedia_sitelink http://pt.wikipedia.org/wiki/Triássico
Q47158-wikipedia_sitelink-17 Q47158 wikipedia_sitelink http://is.wikipedia.org/wiki/Tríastímabilið
Q47158-wikipedia_sitelink-18 Q47158 wikipedia_sitelink http://eo.wikipedia.org/wiki/Triaso
Q47158-wikipedia_sitelink-19 Q47158 wikipedia_sitelink http://ru.wikipedia.org/wiki/Триасовый_период
Q47158-wikipedia_sitelink-2 Q47158 wikipedia_sitelink http://zh.wikipedia.org/wiki/三叠纪
Q47158-wikipedia_sitelink-20 Q47158 wikipedia_sitelink http://sr.wikipedia.org/wiki/Тријас
Q47158-wikipedia_sitelink-21 Q47158 wikipedia_sitelink http://ast.wikipedia.org/wiki/Triásicu
Q47158-wikipedia_sitelink-22 Q47158 wikipedia_sitelink http://fi.wikipedia.org/wiki/Triaskausi
Q47158-wikipedia_sitelink-23 Q47158 wikipedia_sitelink http://uk.wikipedia.org/wiki/Тріасовий_період
Q47158-wikipedia_sitelink-24 Q47158 wikipedia_sitelink http://nn.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-25 Q47158 wikipedia_sitelink http://sah.wikipedia.org/wiki/Триас
Q47158-wikipedia_sitelink-26 Q47158 wikipedia_sitelink http://io.wikipedia.org/wiki/Triaso
Q47158-wikipedia_sitelink-27 Q47158 wikipedia_sitelink http://mzn.wikipedia.org/wiki/تریاس
Q47158-wikipedia_sitelink-28 Q47158 wikipedia_sitelink http://hr.wikipedia.org/wiki/Trijas
Q47158-wikipedia_sitelink-29 Q47158 wikipedia_sitelink http://da.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-3 Q47158 wikipedia_sitelink http://pl.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-30 Q47158 wikipedia_sitelink http://als.wikipedia.org/wiki/Trias_(Geologie)
Q47158-wikipedia_sitelink-31 Q47158 wikipedia_sitelink http://fr.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-32 Q47158 wikipedia_sitelink http://ko.wikipedia.org/wiki/트라이아스기
Q47158-wikipedia_sitelink-33 Q47158 wikipedia_sitelink http://he.wikipedia.org/wiki/טריאס
Q47158-wikipedia_sitelink-34 Q47158 wikipedia_sitelink http://yo.wikipedia.org/wiki/Ìgbà_Tríásíkì
Q47158-wikipedia_sitelink-35 Q47158 wikipedia_sitelink http://lv.wikipedia.org/wiki/Triass
Q47158-wikipedia_sitelink-36 Q47158 wikipedia_sitelink http://it.wikipedia.org/wiki/Triassico
Q47158-wikipedia_sitelink-37 Q47158 wikipedia_sitelink http://gl.wikipedia.org/wiki/Triásico
Q47158-wikipedia_sitelink-38 Q47158 wikipedia_sitelink http://de.wikipedia.org/wiki/Trias_(Geologie)
Q47158-wikipedia_sitelink-39 Q47158 wikipedia_sitelink http://id.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-4 Q47158 wikipedia_sitelink http://eu.wikipedia.org/wiki/Triasiko
Q47158-wikipedia_sitelink-40 Q47158 wikipedia_sitelink http://ml.wikipedia.org/wiki/ട്രയാസ്സിക്
Q47158-wikipedia_sitelink-41 Q47158 wikipedia_sitelink http://ja.wikipedia.org/wiki/三畳紀
Q47158-wikipedia_sitelink-42 Q47158 wikipedia_sitelink http://simple.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-43 Q47158 wikipedia_sitelink http://sh.wikipedia.org/wiki/Trijas
Q47158-wikipedia_sitelink-44 Q47158 wikipedia_sitelink http://sk.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-45 Q47158 wikipedia_sitelink http://en.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-46 Q47158 wikipedia_sitelink http://no.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-47 Q47158 wikipedia_sitelink http://th.wikipedia.org/wiki/ยุคไทรแอสซิก
Q47158-wikipedia_sitelink-48 Q47158 wikipedia_sitelink http://ro.wikipedia.org/wiki/Triasic
Q47158-wikipedia_sitelink-49 Q47158 wikipedia_sitelink http://ca.wikipedia.org/wiki/Triàsic
Q47158-wikipedia_sitelink-5 Q47158 wikipedia_sitelink http://bs.wikipedia.org/wiki/Trijas
Q47158-wikipedia_sitelink-50 Q47158 wikipedia_sitelink http://sl.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-51 Q47158 wikipedia_sitelink http://cs.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-52 Q47158 wikipedia_sitelink http://bg.wikipedia.org/wiki/Триас
Q47158-wikipedia_sitelink-53 Q47158 wikipedia_sitelink http://ka.wikipedia.org/wiki/ტრიასული_სისტემა
Q47158-wikipedia_sitelink-54 Q47158 wikipedia_sitelink http://nds.wikipedia.org/wiki/Trias_(Geologie)
Q47158-wikipedia_sitelink-55 Q47158 wikipedia_sitelink http://lt.wikipedia.org/wiki/Triasas
Q47158-wikipedia_sitelink-56 Q47158 wikipedia_sitelink http://ms.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-57 Q47158 wikipedia_sitelink http://be.wikipedia.org/wiki/Трыясавы_перыяд
Q47158-wikipedia_sitelink-58 Q47158 wikipedia_sitelink http://oc.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-59 Q47158 wikipedia_sitelink http://pms.wikipedia.org/wiki/Triàssich
Q47158-wikipedia_sitelink-6 Q47158 wikipedia_sitelink http://es.wikipedia.org/wiki/Triásico
Q47158-wikipedia_sitelink-60 Q47158 wikipedia_sitelink http://af.wikipedia.org/wiki/Trias
Q47158-wikipedia_sitelink-61 Q47158 wikipedia_sitelink http://zh-yue.wikipedia.org/wiki/三疊紀
Q47158-wikipedia_sitelink-62 Q47158 wikipedia_sitelink http://tt.wikipedia.org/wiki/Триас
Q47158-wikipedia_sitelink-63 Q47158 wikipedia_sitelink http://uz.wikipedia.org/wiki/Trias_sistemasi
Q47158-wikipedia_sitelink-64 Q47158 wikipedia_sitelink http://az.wikipedia.org/wiki/Trias_dövrü
Q47158-wikipedia_sitelink-65 Q47158 wikipedia_sitelink http://kk.wikipedia.org/wiki/Триас_дәуірі
Q47158-wikipedia_sitelink-66 Q47158 wikipedia_sitelink http://cdo.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-67 Q47158 wikipedia_sitelink http://be-x-old.wikipedia.org/wiki/Трыяс
Q47158-wikipedia_sitelink-68 Q47158 wikipedia_sitelink http://la.wikipedia.org/wiki/Triassicum
Q47158-wikipedia_sitelink-69 Q47158 wikipedia_sitelink http://zh-min-nan.wikipedia.org/wiki/Trias-kí
Q47158-wikipedia_sitelink-7 Q47158 wikipedia_sitelink http://ta.wikipedia.org/wiki/டிராசிக்_காலம்
Q47158-wikipedia_sitelink-70 Q47158 wikipedia_sitelink http://chr.wikipedia.org/wiki/ᎤᏂᎾᏅᎪᏥᎵ_ᏧᎾᏔᎾ_ᏘᏲᎭᎵ
Q47158-wikipedia_sitelink-71 Q47158 wikipedia_sitelink http://cy.wikipedia.org/wiki/Triasig
Q47158-wikipedia_sitelink-72 Q47158 wikipedia_sitelink http://tl.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-73 Q47158 wikipedia_sitelink http://sco.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-74 Q47158 wikipedia_sitelink http://fa.wikipedia.org/wiki/تریاس
Q47158-wikipedia_sitelink-75 Q47158 wikipedia_sitelink http://bn.wikipedia.org/wiki/ট্রায়াসিক
Q47158-wikipedia_sitelink-76 Q47158 wikipedia_sitelink http://hy.wikipedia.org/wiki/Տրիասի_ժամանակաշրջան
Q47158-wikipedia_sitelink-77 Q47158 wikipedia_sitelink http://ky.wikipedia.org/wiki/Триас_мезгили
Q47158-wikipedia_sitelink-78 Q47158 wikipedia_sitelink http://zh-classical.wikipedia.org/wiki/三疊紀
Q47158-wikipedia_sitelink-79 Q47158 wikipedia_sitelink http://hi.wikipedia.org/wiki/ट्राइऐसिक_कल्प
Q47158-wikipedia_sitelink-8 Q47158 wikipedia_sitelink http://hu.wikipedia.org/wiki/Triász
Q47158-wikipedia_sitelink-80 Q47158 wikipedia_sitelink http://ar.wikipedia.org/wiki/العصر_الثلاثي
Q47158-wikipedia_sitelink-81 Q47158 wikipedia_sitelink http://wuu.wikipedia.org/wiki/三叠纪
Q47158-wikipedia_sitelink-82 Q47158 wikipedia_sitelink http://vi.wikipedia.org/wiki/Kỷ_Tam_Điệp
Q47158-wikipedia_sitelink-83 Q47158 wikipedia_sitelink http://su.wikipedia.org/wiki/Trias_(periodé)
Q47158-wikipedia_sitelink-84 Q47158 wikipedia_sitelink http://lfn.wikipedia.org/wiki/Triasica
Q47158-wikipedia_sitelink-85 Q47158 wikipedia_sitelink http://war.wikipedia.org/wiki/Triassic
Q47158-wikipedia_sitelink-86 Q47158 wikipedia_sitelink http://tr.wikipedia.org/wiki/Triyas_Devri
Q47158-wikipedia_sitelink-9 Q47158 wikipedia_sitelink http://et.wikipedia.org/wiki/Triias
Q47158-P1014-1 Q47158 P1014 300391257 normal
6 changes: 6 additions & 0 deletions kgtk/tests/data/sample_kgtk_edge_file_no_id.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
node1 label node2
Q1052204 P5456 shengjian-mantou
Q1052204 P646 /m/02qtl2g
Q1052204 description A type of small, pan-fried baozi (steamed buns) which is a specialty of Shanghai
Q1052204 label shengjian mantou
Q1052204 wikipedia_sitelink http://zh.wikipedia.org/wiki/生煎馒头
6 changes: 6 additions & 0 deletions kgtk/tests/data/sample_kgtk_edge_file_with_id.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
node1 label node2 id
Q1052204 P5456 shengjian-mantou 1
Q1052204 P646 /m/02qtl2g 2
Q1052204 description A type of small, pan-fried baozi (steamed buns) which is a specialty of Shanghai 3
Q1052204 label shengjian mantou 4
Q1052204 wikipedia_sitelink http://zh.wikipedia.org/wiki/生煎馒头 5
5 changes: 5 additions & 0 deletions kgtk/tests/data/sample_kgtk_nodes.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
id label type description alias
Q183 'Germany'@en|'Германия'@ru|'Німеччина'@uk item 'sovereign state in central-western Europe'@en|'государство в Центральной Европе'@ru|'країна у Центральній Європі, демократична федеративна республіка'@uk 'FRG'@en|'BRD'@en|'Bundesrepublik Deutschland'@en|'Federal Republic of Germany'@en|'de'@en|'Deutschland'@en|'GER'@en|'BR Deutschland'@en|'Федеративная Республика Германия'@ru|'ФРГ'@ru|'Deutschland'@ru|'Bundesrepublik Deutschland'@ru|'Дойчланд'@ru|'ФРН'@uk|'Федеративна Республіка Німеччина'@uk
Q514 'anatomy'@en|'анатомия'@ru|'анатомія'@uk item 'study of the structure and parts of organisms'@en|'раздел биологии, изучающий строение тела организмов и их частей на уровне выше клеточного'@ru|'наука про форму, будову організму та окремих його органів чи систем'@uk 'general anatomy'@en
Q716 'titanium'@en|'титан'@ru|'титан'@uk item 'chemical element with symbol Ti and atomic number 22'@en|'химический элемент с порядковым номером 22'@ru|'хімічний елемент із атомним номером 22'@uk 'Ti'@en|'element 22'@en|'Ti'@ru|'Ti'@uk
Q901 'scientist'@en|'учёный'@ru|'науковець'@uk item 'person who conducts scientific research into an area of interest'@en|'человек, профессионально использующий научный метод'@ru|'людина, яка професійно використовує науку в дослідженнях'@uk 'natural philosopher'@en|'научный работник'@ru|'дослідник'@uk|'вчений'@uk
Loading

0 comments on commit e8b45e3

Please sign in to comment.