Yl (#19)

* local copy all collections

* debug

* Update walkstac.py

* test corrected data - should be reverted later

* Revert "test corrected data - should be reverted later"

This reverts commit d86baf8.

* Update walkstac.py

* remove pending

* Merge join changes to make indexing work. (#3)

* Use STACObjectType to identifiy objects

* Use STACObjectType to identifiy objects

* commit a run config

* process catalog is an list..

* add runconfig

* Use STACObjectType to identifiy objects

* Added NOTES_DEV.md
fixed some errors in stack for testing.

* Added NOTES_DEV.md
fixed some errors in stack for testing.

* fix some stac data
have collection, why do we search for it?

* Update NOTES_DEV.md

* Update NOTES_DEV.md

* Update walkstac.py

* add jsonld output

* update jsonlds

* generate sitemap

* separate 3 datasets; fix titles and descriptions; auto clear output (#6)

* move sitemaps

* fix ids

* update

* update (#9)

* remove output

* Yl dv demo (#10)

* update

* update

* update

* fix geoshape

* Resolve bounding box (#13)

* update 3 repos

* add def replace

* fix bbox

* fix intervals

* fix intervals

* fix schema

* fix schema

* download github repos

* Update walkstac.py

* update

* add workflow and requirement

* Update requirements.txt

* Update generate_json_files.yml

* Update walkstac.py

* Update generate_json_files.yml

* Update generate_json_files.yml

* add token

* Update generate_json_files.yml

* Update generate_json_files.yml

* Automated commit: Generate folder

* Update generate_json_files.yml

* Update generate_json_files.yml

* add titile

* Update generate_json_files.yml

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* Automated commit: Update data folder

* change keywords to array

* Update generate_json_files.yml

* Automated commit: Update data folder

* Automated commit: Update data folder

---------

Co-authored-by: Ya-Lan Yang <[email protected]>
Co-authored-by: Ya-Lan Yang <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

---------

Co-authored-by: David Valentine <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/generate_json_files.yml

@@ -1,8 +1,8 @@
-name: Run main.py with Config
+name: Update sitemap and jsonlds
 
 on:
   push:
-    branches: [ yl_dv ]
+    branches: [ master ]
   schedule:
     # Runs at 00:00 UTC every day
     - cron: '0 0 * * *'
@@ -15,7 +15,7 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
       with:
-        ref: yl_dv  # Check out the `yl_dv` branch
+        ref: master  # Check out the `yl_dv` branch
 
     - name: Set up Python
       uses: actions/setup-python@v4

archive/spatial.py

@@ -18,14 +18,9 @@
 
 
 def sdo_box(bbox, cells):
-
     doc = {}
-    geo_a = []
-
-    geo = {}
-    geo["@type"] = "GeoShape"
-    geo["box"] = bbox
-    geo_a.append(geo)
+    additional_property = []
+    geo = {"@type": "GeoShape", "box": bbox}
 
     for c in cells:
         geos2 = {}
@@ -34,10 +29,10 @@ def sdo_box(bbox, cells):
         geos2["name"] = "s2Level13"
         geos2["description"] = "S2 cell at level 13"
         geos2["value"] = c
-        geo_a.append(geos2)
-
+        additional_property.append(geos2)
 
     doc['@type'] = 'Place'
-    doc['geo'] = geo_a
+    doc['geo'] = geo
+    doc['additionalProperty'] = additional_property
 
     return doc

data/challenge/NOTES_DEV.md

@@ -0,0 +1,62 @@
+# Notes on fixing issues
+
+# dates
+pending is not a date
+
+## href
+arrays and null is not a valid href
+changed to a valid empty href.
+
+{
+      "rel": "item",
+      "href": [],
+      "type": "text/html",
+      "title": "Link for Model Code"
+    }
+
+https://github.com/earthcube/stacIndexer/blob/yl/data/challenge/neon4cast-stac/forecasts/models/model_items/GLEON_physics.json
+```
+
+should not generate an asset, or may point to example.com
+
+```
+{
+      "rel": "item",
+      "href": "https://github.com/radiantearth",
+      "type": "text/html",
+      "title": "Link for Model Code"
+    }
+## hrefs cannot be arrays
+https://github.com/earthcube/stacIndexer/blob/8a64765c652a85438c54a5b773d0e5a09571ad30/data/challenge/neon4cast-stac/summaries/Aquatics/Daily_Chlorophyll_a/collection.json#L281
+'/Users/valentin/development/dev_earthcube/stacIndexer/data/challenge/neon4cast-stac/summaries/Aquatics/Daily_Chlorophyll_a/collection.json'
+
+['s3://anonymous@bio230014-bucket01/challenges/forecasts/parquet/project_id=neon4cast/duration=P1D/variable=chla?endpoint_override=sdsc.osn.xsede.org', 's3://anonymous@bio230014-bucket01/challenges/forecasts/parquet/project_id=usgsrc4cast/duration=P1D/variable=chla?endpoint_override=sdsc.osn.xsede.org']
+
+
+## bbox should not be lists
+
+"bbox": [
+
+      -156.6194,
+      71.2824,
+      -66.7987,
+      71.2824
+
+  ],
+
+(used two replace in files (
+1:
+"bbox": [
+    [
+
+2:
+
+))
+
+the bbox should be 
+"bbox": [
+      -156.6194,
+      71.2824,
+      -66.7987,
+      71.2824
+  ],

data/challenge/catalog-yl.json

@@ -0,0 +1,38 @@
+{
+    "type": "Catalog",
+    "id": "eco4cast-stac",
+    "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
+    "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
+    "stac_version": "1.0.0",
+    "conformsTo": "conformsTo",
+    "links": [
+        {
+            "rel": "self",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "root",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "neon4cast",
+            "href": "neon4cast-stac/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast-LTREB-reservoirs",
+            "href": "vera4cast-stac/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast-eco4cast",
+            "href": "usgsrc4cast-stac/catalog.json"
+        }
+    ]
+}

data/challenge/catalog.json

@@ -1,38 +1,38 @@
 {
-  "type": "Catalog",
-  "id": "eco4cast-stac",
-  "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
-  "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
-  "stac_version": "1.0.0",
-  "conformsTo": "conformsTo",
-  "links": [
-    {
-      "rel": "self",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "root",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "neon4cast",
-      "href": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast",
-      "href": "https://raw.githubusercontent.com/LTREB-reservoirs/vera4cast/main/catalog/catalog.json"
-    },
+    "type": "Catalog",
+    "id": "eco4cast-stac",
+    "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
+    "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
+    "stac_version": "1.0.0",
+    "conformsTo": "conformsTo",
+    "links": [
         {
-      "rel": "child",
-      "type": "application/json",
-      "title": "vera4cast",
-      "href": "https://raw.githubusercontent.com/eco4cast/usgsrc4cast-ci/main/catalog/catalog.json"
-    }
-  ]
-}
+            "rel": "self",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "root",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "neon4cast",
+            "href": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast",
+            "href": "https://raw.githubusercontent.com/LTREB-reservoirs/vera4cast/main/catalog/catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "vera4cast",
+            "href": "https://raw.githubusercontent.com/eco4cast/usgsrc4cast-ci/main/catalog/catalog.json"
+        }
+    ]
+}

data/challenge/catalogEDITED.json

@@ -1,27 +1,27 @@
 {
-  "type": "Catalog",
-  "id": "eco4cast-stac",
-  "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
-  "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
-  "stac_version": "1.0.0",
-  "conformsTo": "conformsTo",
-  "links": [
-    {
-      "rel": "self",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "root",
-      "type": "application/json",
-      "href": "catalog.json"
-    },
-    {
-      "rel": "child",
-      "type": "application/json",
-      "title": "neon4cast",
-      "href": "neon4cast-ci/catalog/scores/collection.json",
-      "hrefold": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
-    }
-  ]
-}
+    "type": "Catalog",
+    "id": "eco4cast-stac",
+    "title": "Ecological Forecasting Initiative Forecasting Challenges Catalog",
+    "description": "A STAC (Spatiotemporal Asset Catalog) describing forecasts and forecast scores for ecological forecasting challenges",
+    "stac_version": "1.0.0",
+    "conformsTo": "conformsTo",
+    "links": [
+        {
+            "rel": "self",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "root",
+            "type": "application/json",
+            "href": "catalog.json"
+        },
+        {
+            "rel": "child",
+            "type": "application/json",
+            "title": "neon4cast",
+            "href": "neon4cast-ci/catalog/scores/collection.json",
+            "hrefold": "https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/catalog.json"
+        }
+    ]
+}

data/challenge/neon4cast-stac/R/catalog-common.R

@@ -0,0 +1,4 @@
+conformsTo = list(
+  "https=//api.stacspec.org/v1.0.0-rc.1/collections",
+  "https=//api.stacspec.org/v1.0.0-rc.1/core"
+)