Python_client_issue_3

.devcontainer.json

@@ -0,0 +1,31 @@
+{
+	"name": "Python 3.8",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": ".."
+	},
+    "customizations": {
+        "vscode": {
+            "settings": { 
+				"python.defaultInterpreterPath": "/usr/local/bin/python",
+				"python.linting.enabled": true,
+				"python.linting.pylintEnabled": false,
+				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+				"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+				"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+				"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+				"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+				"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+				"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+			},
+            "extensions": [
+                "ms-python.python",
+                "ms-python.vscode-pylance"
+            ]
+        }
+    },
+	"postCreateCommand": "pip install -e stac-task/"
+}
+

.gitignore

@@ -0,0 +1,5 @@
+.DS_store
+.pytest_cache
+**.pytest_cache**
+**__pycache__**
+data/**

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "stac-task"]
+	path = stac-task
+	url = [email protected]:stac-utils/stac-task.git

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python.analysis.extraPaths": [
+        "./stac-task"
+    ]
+}

Dockerfile

@@ -0,0 +1 @@
+FROM mcr.microsoft.com/devcontainers/python:3.8

client/README.md

@@ -0,0 +1,48 @@
+# MLRD client
+Python client which can read a MLRD training dataset catalog and provide an iterator which for each iteration loads a label chip and its source imagery and returns a numpy array for use in training
+
+## Install Dependencies
+
+With dev dependencies:
+```shell
+poetry install
+```
+
+Without dev dependencies:
+```shell
+poetry install --no-dev
+```
+
+## Update Dependencies
+
+```shell
+poetry update
+```
+
+## Add new Dependency
+
+```shell
+poetry add requests
+```
+Development-only dependency:
+```shell
+poetry add --dev pytest
+```
+
+## Build project
+
+```shell
+poetry build
+```
+
+## Lint project
+
+```shell
+poetry run flake8
+```
+
+## Run Tests
+
+```shell
+poetry run pytest tests
+```

client/client/myFunction.py

@@ -0,0 +1,24 @@
+from math import radians, cos, sin, asin, sqrt
+
+def haversine(lon1: float, lat1: float, lon2: float, lat2: float) -> float:
+    """
+    Calculate the great circle distance between two points on the 
+    earth (specified in decimal degrees), returns the distance in
+    kilometers.
+    All arguments must be of equal length.
+    :param lon1: longitude of first place
+    :param lat1: latitude of first place
+    :param lon2: longitude of second place
+    :param lat2: latitude of second place
+    :return: distance in kilometers between the two sets of coordinates
+    """
+    # Convert decimal degrees to radians
+    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
+
+    # Haversine formula
+    dlon = lon2 - lon1
+    dlat = lat2 - lat1
+    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
+    c = 2 * asin(sqrt(a))
+    r = 6371 # Radius of earth in kilometers
+    return c * r

client/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "client"
+version = "0.1.0"
+description = ""
+authors = ["doug-newman-nasa <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+torch = "^2.1.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

client/tests/test_myFunction.py

@@ -0,0 +1,9 @@
+from client import myFunction
+
+import unittest
+
+def test_haversine():
+    # Amsterdam to Berlin
+    assert myFunction.haversine(
+        4.895168, 52.370216, 13.404954, 52.520008
+    ) == 576.6625818456291

examples/landcovernet/input/landcovernet.json

@@ -0,0 +1,89 @@
+{
+    "type": "FeatureCollection",
+    "process": {
+        "input_collections": [
+            "sentinel-2-l2a"
+        ],
+        "workflow": "cog-archive",
+        "upload_options": {
+            "path_template": "s3://sentinel-cogs/${collection}/${grid:utm_zone}/${grid:latitude_band}/${grid:grid_square}/${year}/${month}/${id}",
+            "public_assets": "ALL",
+            "collections": {
+                "sentinel-2-l2a": "$[?(@.id =~ 'S2[AB].*')]"
+            },
+            "headers": {
+                "CacheControl": "public, max-age=31536000, immutable"
+            }
+        },
+        "tasks": [
+            {   
+                "name": "inference",
+                "parameters": {
+                    "test": "foobar"
+                }
+            }
+        ]
+    },
+    "features": [
+        {
+            "id": "ref_landcovernet_sa_v1_labels_17NRB_00",
+            "datetime": "2018-07-01T00:00:00Z",
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [
+                            -77.81057792539683,
+                            0.9392721120508254
+                        ],
+                        [
+                            -77.81055668144066,
+                            0.962397053731636
+                        ],
+                        [
+                            -77.83353035617907,
+                            0.9624186203295519
+                        ],
+                        [
+                            -77.83355144774016,
+                            0.9392931603458728
+                        ],
+                        [
+                            -77.81057792539683,
+                            0.9392721120508254
+                        ]
+                    ]
+                ]
+            },
+            "assets": {
+                "labels": {
+                    "href": "https://radiantearth.blob.core.windows.net/mlhub/landcovernet_sa/data/v1.0/2018/17NRB/00/17NRB_00_2018_LC_10m.tif",
+                    "title": "Land Cover"
+                },
+                "source_dates": {
+                    "href": "https://radiantearth.blob.core.windows.net/mlhub/landcovernet_sa/data/v1.0/2018/17NRB/00/17NRB_00_labeling_dates.csv",
+                    "type": "text/csv",
+                    "title": "Source Imagery Dates"
+                },
+                "documentation": {
+                    "href": "https://radiantearth.blob.core.windows.net/mlhub/landcovernet_sa/Documentation.pdf",
+                    "type": "application/pdf"
+                }
+            },
+            "collection": "ref_landcovernet_sa_v1_labels",
+            "properties": {
+                "datetime": "2018-07-01T00:00:00Z"
+            },
+            "bbox": [
+                -77.83355144774016,
+                0.9392721120508254,
+                -77.81055668144066,
+                0.9624186203295519
+            ],
+            "type": "Feature",
+            "stac_version": "1.0.0",
+            "links": [
+            ]
+        }
+    ]
+}

examples/landcovernet/main.py

@@ -0,0 +1,77 @@
+import json
+import torch
+from stactask import Task
+from typing import List, Dict, Any
+
+
+class LandCoverClassificationModel(torch.nn.Module):
+    def __init__(self, checkpoint=None):
+        self.checkpoint = checkpoint
+        pass
+
+
+class TrainTask(Task):
+    name = "train"
+    description = "This task trains the model"
+
+    def batches(self, batch_size: int):
+        l = len(self.items_as_dicts)
+        for i in range(0, l, batch_size):
+            yield self.items_as_dicts[i:min(i+batch_size, l)]
+
+    def process(self, batch_size=100, checkpoint=None) -> List[Dict[str, Any]]:
+        model = LandCoverClassificationModel()
+
+        for i, batch in enumerate(self.batches(batch_size=batch_size)):
+            print(f"Working on Batch {i+1}")
+
+        return []
+
+
+class InferenceTask(Task):
+    name = "inference"
+    description = "This task creates inferences for the model"
+
+    def process(self, **kwargs) -> List[Dict[str, Any]]:
+        print(kwargs)
+        input_item = self.items_as_dicts[0]
+
+        """
+            TODO:
+            - Take the geometry and datetime from the given input STAC item
+            - Load the imagery required for this model using something like stackstac
+            - Transform the imagery as needed
+            - Run the model inferencing code using the transformed imagery
+            - Transform the inference output as needed
+            - Upload the transformed inference output to some cloud object storage
+            - Return the output STAC item with the inference item as an asset
+        """
+
+        output_item = {
+            "type": "Feature",
+            "id": input_item["id"],
+            "stac_version": input_item["stac_version"],
+            "geometry": input_item["geometry"],
+            "properties": {
+                "datetime": input_item["properties"]["datetime"]
+            },
+            "links": [],
+            "assets": {
+                "inference": {
+                    "href": "s3://foobar/inferences/id.tif",
+                    "type": "image/tiff; application=geotiff; profile=cloud-optimized"
+                }
+            }
+        }
+        return [
+            output_item
+        ]
+
+
+if __name__ == "__main__":
+    with open("examples/landcovernet/input/landcovernet.json", "r") as f:
+        item_collection = json.load(f)
+
+    t = TrainTask(item_collection)
+    items = t.process(batch_size=100)
+    print(items)