Task improvements

dev/Dockerfile

@@ -8,15 +8,21 @@ RUN apt-get update && \
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
-# Install tile2net (may take up to 30 mins)
-RUN git clone https://github.com/VIDA-NYU/tile2net.git
-RUN python -m pip install ./tile2net
+ARG TASKS
 
 COPY ./setup.py /opt/geoinsight-server/setup.py
 COPY ./manage.py /opt/geoinsight-server/manage.py
 COPY ./geoinsight /opt/geoinsight-server/geoinsight
+
 RUN pip install large-image[all] large-image-converter --find-links https://girder.github.io/large_image_wheels
-RUN pip install --editable /opt/geoinsight-server[dev,tasks]
+RUN pip install --editable /opt/geoinsight-server[dev]
+
+RUN if [ "$TASKS" = "1" ]; \
+    then pip install --editable /opt/geoinsight-server[tasks]; \
+    # Install tile2net (may take up to 30 mins)
+    git clone https://github.com/VIDA-NYU/tile2net.git; \
+    python -m pip install ./tile2net; \
+    fi
 
 # Use a directory name which will never be an import name, as isort considers this as first-party.
 WORKDIR /opt/geoinsight-server

docker-compose.override.yml

@@ -25,6 +25,8 @@ services:
     build:
       context: .
       dockerfile: ./dev/Dockerfile
+      args:
+        TASKS: 1
     command:
       [
         "celery",

geoinsight/core/management/commands/ingest.py

@@ -13,7 +13,7 @@
 from django.core.management.base import BaseCommand
 import pooch
 
-from geoinsight.core.models import Chart, Dataset, DatasetTag, FileItem, Project
+from geoinsight.core.models import Chart, Dataset, FileItem, Project
 
 DATA_FOLDER = Path(os.environ.get('INGEST_BIND_MOUNT_POINT', 'sample_data'))
 DOWNLOADS_FOLDER = Path(DATA_FOLDER, 'downloads')
@@ -401,10 +401,5 @@ def ingest_datasets(
                     )
                 )
 
-            self.ingest_dataset_tags(dataset_for_conversion, dataset.get('tags'))
+            dataset_for_conversion.set_tags(dataset.get('tags'))
             dataset_for_conversion.set_owner(superuser)
-
-    def ingest_dataset_tags(self, dataset, tags):
-        for tag in tags:
-            DatasetTag.objects.get_or_create(tag=tag)
-        dataset.tags.set(DatasetTag.objects.filter(tag__in=tags))

geoinsight/core/models/dataset.py

@@ -58,6 +58,12 @@ def set_owner(self, user: User):
         ).delete()
         assign_perm('owner', user, self)
 
+    @transaction.atomic()
+    def set_tags(self, tags: [str]):
+        for tag in tags:
+            DatasetTag.objects.get_or_create(tag=tag)
+        self.tags.set(DatasetTag.objects.filter(tag__in=tags))
+
     def spawn_conversion_task(
         self,
         layer_options=None,

geoinsight/core/rest/analytics.py

@@ -28,6 +28,8 @@ class AnalyticsViewSet(ReadOnlyModelViewSet):
     def list_types(self, request, project_id: int, **kwargs):
         serialized = []
         for analysis_type in analysis_types:
+            if not analysis_type.is_enabled():
+                continue
             instance = analysis_type()
             filtered_input_options = {}
             for k, v in instance.get_input_options().items():
@@ -79,7 +81,7 @@ def run(self, request, project_id: int, task_type: str, **kwargs):
         analysis_type_class = next(
             iter(at for at in analysis_types if at().db_value == task_type), None
         )
-        if analysis_type_class is None:
+        if analysis_type_class is None or not analysis_type_class.is_enabled():
             return Response(f'Analysis type "{task_type}" not found', status=404)
         result = analysis_type_class().run_task(project, **request.data)
         return Response(

geoinsight/core/rest/project.py

@@ -2,7 +2,6 @@
 from typing import Any
 
 from django.contrib.auth.models import User
-from django.http import HttpResponse
 from drf_yasg.utils import swagger_auto_schema
 from rest_framework.decorators import action
 from rest_framework.request import Request
@@ -12,7 +11,6 @@
 from geoinsight.core.models import Project
 from geoinsight.core.rest.access_control import GuardianFilter, GuardianPermission
 from geoinsight.core.rest.serializers import ProjectPermissionsSerializer, ProjectSerializer
-from geoinsight.core.tasks.osmnx import load_roads
 
 
 class ProjectViewSet(ModelViewSet):
@@ -51,14 +49,3 @@ def permissions(self, request: Request, *args: Any, **kwargs: Any):
         )
 
         return Response(ProjectSerializer(project).data, status=200)
-
-    # TODO: This should be a POST
-    @action(
-        detail=True,
-        methods=['get'],
-        url_path=r'load_roads/(?P<location>.+)',
-    )
-    def load_roads(self, request, location, **kwargs):
-        project = self.get_object()
-        load_roads.delay(project.id, location)
-        return HttpResponse('Task spawned successfully.', status=200)

geoinsight/core/tasks/analytics/__init__.py

@@ -1,7 +1,15 @@
+from .create_road_network import CreateRoadNetwork
 from .flood_network_failure import FloodNetworkFailure
 from .flood_simulation import FloodSimulation
 from .geoai_segmentation import GeoAISegmentation
 from .network_recovery import NetworkRecovery
-from .segment_curbs import SegmentCurbs
+from .tile2net_segmentation import Tile2NetSegmentation
 
-__all__ = [FloodSimulation, FloodNetworkFailure, NetworkRecovery, SegmentCurbs, GeoAISegmentation]
+__all__ = [
+    FloodSimulation,
+    FloodNetworkFailure,
+    NetworkRecovery,
+    Tile2NetSegmentation,
+    GeoAISegmentation,
+    CreateRoadNetwork,
+]

geoinsight/core/tasks/analytics/analysis_type.py

@@ -10,6 +10,10 @@ def __init__(self, *args):
         self.output_types = {}
         self.attribution = 'Kitware, Inc.'
 
+    @abstractmethod
+    def is_enabled(self):
+        raise NotImplementedError
+
     @abstractmethod
     def get_input_options(self):
         raise NotImplementedError

geoinsight/core/tasks/analytics/create_road_network.py

@@ -0,0 +1,149 @@
+import datetime
+
+from celery import shared_task
+from django.conf import settings
+from django.contrib.gis.geos import LineString, Point
+
+from geoinsight.core.models import (
+    Dataset,
+    Layer,
+    LayerFrame,
+    Network,
+    NetworkEdge,
+    NetworkNode,
+    TaskResult,
+    VectorData,
+)
+from geoinsight.core.tasks.data import create_vector_features
+from geoinsight.core.tasks.networks import geojson_from_network
+
+from .analysis_type import AnalysisType
+
+
+class CreateRoadNetwork(AnalysisType):
+    def __init__(self):
+        super().__init__(self)
+        self.name = 'Create Road Network'
+        self.description = 'Leverage OSMnx to create a road network for a target location'
+        self.db_value = 'create_road_network'
+        self.input_types = {'location': 'string'}
+        self.output_types = {'roads': 'Dataset'}
+        self.attribution = 'https://osmnx.readthedocs.io/en/stable/'
+
+    @classmethod
+    def is_enabled(cls):
+        return settings.ENABLE_TASK_CREATE_ROAD_NETWORK
+
+    def get_input_options(self):
+        return {'location': []}
+
+    def run_task(self, project, **inputs):
+        location = inputs.get('location')
+        result = TaskResult.objects.create(
+            name=f'Create Road Network for {location}',
+            task_type=self.db_value,
+            inputs=inputs,
+            project=project,
+            status='Initializing task...',
+        )
+        create_road_network.delay(result.id)
+        return result
+
+
+def metadata_for_row(row):
+    return {
+        k: v
+        for k, v in row.to_dict().items()
+        if k not in ['osmid', 'geometry', 'ref'] and str(v) != 'nan'
+    }
+
+
+@shared_task
+def create_road_network(result_id):
+    import osmnx
+
+    result = TaskResult.objects.get(id=result_id)
+    try:
+        location = result.inputs.get('location')
+        if location is None:
+            raise ValueError('location not provided')
+
+        result.write_status('Fetching road data via OSMnx...')
+        roads = osmnx.graph_from_place(location, network_type='drive')
+        road_nodes, road_edges = osmnx.graph_to_gdfs(roads)
+
+        result.write_status('Saving results to database...')
+        name = f'Road Network for {location}'
+        existing_count = Dataset.objects.filter(name__icontains=name).count()
+        if existing_count:
+            name += f' ({existing_count + 1})'
+        dataset = Dataset.objects.create(
+            name=name,
+            description='Network generated by OSMnx from OpenStreetMap data',
+            category='transportation',
+            metadata={'creation_time': datetime.datetime.now(datetime.UTC).isoformat()},
+        )
+        dataset.set_tags(['analytics', 'osmnx', 'network'])
+        vector_data = VectorData.objects.create(
+            name=f'{location} Roads Vector Data', dataset=dataset
+        )
+        layer = Layer.objects.create(name=f'{location} Roads', dataset=dataset)
+        LayerFrame.objects.create(name='Roads', layer=layer, vector=vector_data)
+        network = Network.objects.create(
+            name=f'{location} Road Network',
+            category='roads',
+            vector_data=vector_data,
+            metadata={'source': 'Created with OSMnx'},
+        )
+
+        for _, edge_data in road_edges.iterrows():
+            edge_geom = edge_data['geometry'].coords
+            start = edge_geom[0]
+            end = edge_geom[-1]
+            edge_name = edge_data['name']
+            if str(edge_name) == 'nan' or len(str(edge_name)) < 2:
+                # If name is invalid, write new name string
+                edge_name = 'Unnamed Road at {:0.4f}/{:0.4f}'.format(
+                    *edge_geom[int(len(edge_geom) / 2)]
+                )
+
+            start_node_data = road_nodes.loc[
+                (road_nodes['x'] == start[0]) & (road_nodes['y'] == start[1])
+            ].iloc[0]
+            end_node_data = road_nodes.loc[
+                (road_nodes['x'] == end[0]) & (road_nodes['y'] == end[1])
+            ].iloc[0]
+
+            start_node, created = NetworkNode.objects.get_or_create(
+                network=network,
+                name='{:0.5f}/{:0.5f}'.format(*start),
+                location=Point(*start),
+            )
+            start_node.metadata = metadata_for_row(start_node_data)
+            start_node.save()
+            end_node, created = NetworkNode.objects.get_or_create(
+                network=network,
+                name='{:0.5f}/{:0.5f}'.format(*end),
+                location=Point(*end),
+            )
+            end_node.metadata = metadata_for_row(end_node_data)
+            end_node.save()
+            edge, created = NetworkEdge.objects.get_or_create(
+                network=network,
+                name=edge_name,
+                directed=edge_data['oneway'],
+                from_node=start_node,
+                to_node=end_node,
+                line_geometry=LineString(*[Point(*p) for p in edge_geom]),
+            )
+            edge.metadata = metadata_for_row(edge_data)
+            edge.save()
+
+        vector_data.write_geojson_data(geojson_from_network(dataset))
+        create_vector_features(vector_data)
+        vector_data.get_summary()
+
+        result.outputs = dict(roads=dataset.id)
+    except Exception as e:
+        result.error = str(e)
+    result.complete()

geoinsight/core/tasks/analytics/flood_network_failure.py

@@ -1,6 +1,7 @@
 import math
 
 from celery import shared_task
+from django.conf import settings
 from django_large_image import tilesource, utilities
 import numpy
 
@@ -28,6 +29,10 @@ def __init__(self):
         self.output_types = {'failures': 'network_animation'}
         self.attribution = 'Northeastern University & Kitware, Inc.'
 
+    @classmethod
+    def is_enabled(cls):
+        return settings.ENABLE_TASK_FLOOD_NETWORK_FAILURE
+
     def get_input_options(self):
         return {
             'network': Network.objects.all(),

geoinsight/core/tasks/analytics/flood_simulation.py

@@ -5,6 +5,7 @@
 import tempfile
 
 from celery import shared_task
+from django.conf import settings
 from django.core.files.base import ContentFile
 
 from geoinsight.core.models import Chart, Colormap, Dataset, FileItem, LayerStyle, TaskResult
@@ -33,6 +34,10 @@ def __init__(self):
         self.output_types = {'flood': 'Dataset'}
         self.attribution = 'Northeastern University'
 
+    @classmethod
+    def is_enabled(cls):
+        return settings.ENABLE_TASK_FLOOD_SIMULATION
+
     def get_input_options(self):
         return {
             'time_period': ['2030-2050'],
@@ -199,6 +204,7 @@ def flood_simulation(result_id):
                 category='flood',
                 metadata=metadata,
             )
+            dataset.set_tags(['analytics', 'flood', 'simulation'])
             file_item = FileItem.objects.create(
                 name=output_path.name,
                 dataset=dataset,