Merge pull request #11054 from camptocamp/geo-7032-cog

Add support of COG layers

.github/workflows/main.yaml

@@ -80,6 +80,9 @@ jobs:
 
       - name: Environment information
         run: c2cciutils-env
+        env:
+          # To display the content of the github object
+          GITHUB_EVENT: ${{ toJson(github) }}
 
       - run: python3 -m pip install --user --requirement=requirements.txt
 
@@ -90,22 +93,50 @@ jobs:
         if: always()
       - run: make build-config
 
-        # Build and lint QGIS images
+      # Build and lint QGIS images
       - run: QGIS_VERSION=3.34-gdal3.8 make build-qgisserver
       - run: QGIS_VERSION=3.34-gdal3.8 make prospector-qgisserver
 
-        # Tests
       - run: make preparetest
-      - run: c2cciutils-docker-logs
+
+      # Auto generate Alembic script
+      - run: |
+          docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main check \
+            || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=main \
+            revision --autogenerate --message='${{ github.event.pull_request.title }}'
+          docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static check \
+            || docker compose exec tests alembic --config=/opt/c2cgeoportal/commons/alembic.ini --name=static \
+            revision --autogenerate --message='${{ github.event.pull_request.title }}'
+          sudo chmod go+rw -R commons/c2cgeoportal_commons/alembic/
+          # Replace 'main' by schema and 'main. by f'{schema}.
+          sed -i "s/'main'/schema/g" commons/c2cgeoportal_commons/alembic/*/*.py
+          sed -i "s/'main\.'/f'{schema}./g" commons/c2cgeoportal_commons/alembic/*/*.py
+          # Replace 'main_static' by staticschema and 'main_static. by f'{staticschema}.
+          sed -i "s/'main_static'/staticschema/g" commons/c2cgeoportal_commons/alembic/*/*.py
+          sed -i "s/'main_static\.'/f'{staticschema}./g" commons/c2cgeoportal_commons/alembic/*/*.py
+          git checkout commons/c2cgeoportal_commons/alembic/main/ee25d267bf46_main_interface_desktop.py
+          git checkout commons/c2cgeoportal_commons/alembic/main/415746eb9f6_changes_for_v2.py
+          git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py
+          pre-commit run --files commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py || true
+          git add commons/c2cgeoportal_commons/alembic/main/*.py commons/c2cgeoportal_commons/alembic/static/*.py
+          git diff --staged --patch > /tmp/alembic.patch
+          git diff --staged --exit-code
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Add Alembic upgrade script.patch
+          path: /tmp/alembic.patch
+          retention-days: 1
         if: failure()
+
+      # Tests
+      - run: c2cciutils-docker-logs
       - run: make tests-commons
       - run: c2cciutils-docker-logs
       - run: make tests-geoportal
       - run: c2cciutils-docker-logs
       - run: make tests-admin
       - run: c2cciutils-docker-logs
       - run: make tests-qgisserver
-      - run: c2cciutils-docker-logs
       - run: c2cciutils-docker-logs
         if: always()
       - run: docker compose down

admin/c2cgeoportal_admin/routes.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2023, Camptocamp SA
+# Copyright (c) 2017-2024, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -57,6 +57,7 @@ def includeme(config):
     from c2cgeoportal_commons.models.main import (  # pylint: disable=import-outside-toplevel
         Functionality,
         Interface,
+        LayerCOG,
         LayerGroup,
         LayerVectorTiles,
         LayerWMS,
@@ -78,6 +79,7 @@ def includeme(config):
         ("layers_wms", LayerWMS),
         ("layers_wmts", LayerWMTS),
         ("layers_vectortiles", LayerVectorTiles),
+        ("layers_cog", LayerCOG),
         ("ogc_servers", OGCServer),
         ("restriction_areas", RestrictionArea),
         ("users", User),

admin/c2cgeoportal_admin/views/layers_cog.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2017-2024, Camptocamp SA
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The views and conclusions contained in the software and documentation are those
+# of the authors and should not be interpreted as representing official policies,
+# either expressed or implied, of the FreeBSD Project.
+
+
+from functools import partial
+from typing import Optional, cast
+
+import sqlalchemy
+import sqlalchemy.orm.query
+from c2cgeoform.schema import GeoFormSchemaNode
+from c2cgeoform.views.abstract_views import (
+    DeleteResponse,
+    GridResponse,
+    IndexResponse,
+    ListField,
+    ObjectResponse,
+    SaveResponse,
+)
+from deform.widget import FormWidget
+from pyramid.httpexceptions import HTTPNotFound
+from pyramid.view import view_config, view_defaults
+
+from c2cgeoportal_admin import _
+from c2cgeoportal_admin.schemas.interfaces import interfaces_schema_node
+from c2cgeoportal_admin.schemas.metadata import metadata_schema_node
+from c2cgeoportal_admin.schemas.restriction_areas import restrictionareas_schema_node
+from c2cgeoportal_admin.schemas.treeitem import parent_id_node
+from c2cgeoportal_admin.views.layers import LayerViews
+from c2cgeoportal_commons.lib.literal import Literal
+from c2cgeoportal_commons.models.main import LayerCOG, LayerGroup
+
+_list_field = partial(ListField, LayerCOG)  # type: ignore[var-annotated]
+
+
+base_schema = GeoFormSchemaNode(LayerCOG, widget=FormWidget(fields_template="layer_fields"))
+base_schema.add(metadata_schema_node(LayerCOG.metadatas, LayerCOG))
+base_schema.add(interfaces_schema_node(LayerCOG.interfaces))
+base_schema.add(restrictionareas_schema_node(LayerCOG.restrictionareas))
+base_schema.add_unique_validator(LayerCOG.name, LayerCOG.id)
+base_schema.add(parent_id_node(LayerGroup))
+
+
+@view_defaults(match_param="table=layers_cog")
+class LayerCOGViews(LayerViews[LayerCOG]):
+    """The vector tiles administration view."""
+
+    _list_fields = (
+        LayerViews._list_fields  # typer: ignore[misc] # pylint: disable=protected-access
+        + [_list_field("url")]
+        + LayerViews._extra_list_fields  # pylint: disable=protected-access
+    )
+
+    _id_field = "id"
+    _model = LayerCOG
+    _base_schema = base_schema
+
+    def _base_query(self) -> sqlalchemy.orm.query.Query[LayerCOG]:
+        return super()._sub_query(self._request.dbsession.query(LayerCOG).distinct())
+
+    def _sub_query(
+        self, query: Optional[sqlalchemy.orm.query.Query[LayerCOG]]
+    ) -> sqlalchemy.orm.query.Query[LayerCOG]:
+        del query
+        return self._base_query()
+
+    @view_config(route_name="c2cgeoform_index", renderer="../templates/index.jinja2")  # type: ignore[misc]
+    def index(self) -> IndexResponse:
+        return super().index()
+
+    @view_config(route_name="c2cgeoform_grid", renderer="fast_json")  # type: ignore[misc]
+    def grid(self) -> GridResponse:
+        return super().grid()
+
+    def schema(self) -> GeoFormSchemaNode:
+        try:
+            obj = self._get_object()
+        except HTTPNotFound:
+            obj = None
+
+        schema = cast(GeoFormSchemaNode, self._base_schema.clone())
+        if obj is not None:
+            schema["url"].description = Literal(
+                _("{}<br>Current runtime value is: {}").format(
+                    schema["url"].description,
+                    obj.url_description(self._request),
+                )
+            )
+        return schema
+
+    @view_config(route_name="c2cgeoform_item", request_method="GET", renderer="../templates/edit.jinja2")  # type: ignore[misc]
+    def view(self) -> ObjectResponse:
+        if self._is_new():
+            dbsession = self._request.dbsession
+            default_cog = LayerCOG.get_default(dbsession)
+            if default_cog:
+                return self.copy(default_cog, excludes=["name", "url"])
+        return super().edit()
+
+    @view_config(route_name="c2cgeoform_item", request_method="POST", renderer="../templates/edit.jinja2")  # type: ignore[misc]
+    def save(self) -> SaveResponse:
+        return super().save()
+
+    @view_config(route_name="c2cgeoform_item", request_method="DELETE", renderer="fast_json")  # type: ignore[misc]
+    def delete(self) -> DeleteResponse:
+        return super().delete()
+
+    @view_config(  # type: ignore[misc]
+        route_name="c2cgeoform_item_duplicate", request_method="GET", renderer="../templates/edit.jinja2"
+    )
+    def duplicate(self) -> ObjectResponse:
+        return super().duplicate()

admin/tests/__init__.py

@@ -9,13 +9,15 @@
 
 
 def get_test_default_layers(dbsession, default_ogc_server):
-    from c2cgeoportal_commons.models.main import LayerVectorTiles, LayerWMS, LayerWMTS
-
-    default_wms = LayerWMS("wms-defaults")
-    default_wms.ogc_server = default_ogc_server
-    default_wms.time_widget = "datepicker"
-    default_wms.time_mode = "value"
-    dbsession.add(default_wms)
+    from c2cgeoportal_commons.models.main import LayerCOG, LayerVectorTiles, LayerWMS, LayerWMTS
+
+    default_wms = None
+    if default_ogc_server:
+        default_wms = LayerWMS("wms-defaults")
+        default_wms.ogc_server = default_ogc_server
+        default_wms.time_widget = "datepicker"
+        default_wms.time_mode = "value"
+        dbsession.add(default_wms)
     default_wmts = LayerWMTS("wmts-defaults")
     default_wmts.url = "https:///wmts.geo.admin_default.ch.org?service=wms&request=GetCapabilities"
     default_wmts.layer = "default"
@@ -24,8 +26,11 @@ def get_test_default_layers(dbsession, default_ogc_server):
     default_vectortiles = LayerVectorTiles("vectortiles-defaults")
     default_vectortiles.style = "https://vectortiles-staging.geoportail.lu/styles/roadmap/style.json"
     dbsession.add(default_vectortiles)
+    default_cog = LayerCOG("cog-defaults")
+    default_cog.url = "https://example.com/image.tiff"
+    dbsession.add(default_cog)
     dbsession.flush()
-    return {"wms": default_wms, "wmts": default_wmts, "vectortiles": default_vectortiles}
+    return {"wms": default_wms, "wmts": default_wmts, "vectortiles": default_vectortiles, "cog": default_cog}
 
 
 def factory_build_layers(layer_builder, dbsession, add_dimension=True):

admin/tests/conftest.py

@@ -1,9 +1,15 @@
+from typing import Any
+
+import pyramid.request
 import pytest
 import sqlalchemy.exc
 import transaction
 from pyramid import testing
 from pyramid.paster import bootstrap
+from pyramid.router import Router
+from pyramid.scripting import AppEnvironment
 from sqlalchemy.exc import DBAPIError
+from sqlalchemy.orm import Session, SessionTransaction
 from webtest import TestApp as WebTestApp  # Avoid warning with pytest
 
 from c2cgeoportal_commons.testing import generate_mappers, get_engine, get_session_factory, get_tm_session
@@ -12,7 +18,7 @@
 
 @pytest.fixture(scope="session")
 @pytest.mark.usefixtures("settings")
-def dbsession(settings):
+def dbsession(settings: dict[str, Any]) -> Session:
     generate_mappers()
     engine = get_engine(settings)
     session_factory = get_session_factory(engine)
@@ -23,7 +29,7 @@ def dbsession(settings):
 
 @pytest.fixture(scope="function")
 @pytest.mark.usefixtures("dbsession")
-def transact(dbsession):
+def transact(dbsession: Session) -> SessionTransaction:
     t = dbsession.begin_nested()
     yield t
     try:
@@ -33,29 +39,29 @@ def transact(dbsession):
     dbsession.expire_all()
 
 
-def raise_db_error(_):
+def raise_db_error(_: Any) -> None:
     raise DBAPIError("this is a test !", None, None)
 
 
 @pytest.fixture(scope="function")
 @pytest.mark.usefixtures("dbsession")
-def raise_db_error_on_query(dbsession):
+def raise_db_error_on_query(dbsession: Session) -> None:
     query = dbsession.query
     dbsession.query = raise_db_error
     yield
     dbsession.query = query
 
 
 @pytest.fixture(scope="session")
-def app_env():
+def app_env() -> AppEnvironment:
     file_name = "/opt/c2cgeoportal/admin/tests/tests.ini"
     with bootstrap(file_name) as env:
         yield env
 
 
 @pytest.fixture(scope="session")
 @pytest.mark.usefixtures("app_env", "dbsession")
-def app(app_env, dbsession):
+def app(app_env: AppEnvironment, dbsession: Session) -> Router:
     config = testing.setUp(registry=app_env["registry"])
     config.add_request_method(lambda request: dbsession, "dbsession", reify=True)
     config.add_route("user_add", "user_add")
@@ -68,12 +74,12 @@ def app(app_env, dbsession):
 
 @pytest.fixture(scope="session")
 @pytest.mark.usefixtures("app_env")
-def settings(app_env):
+def settings(app_env: AppEnvironment) -> Any:
     yield app_env.get("registry").settings
 
 
 @pytest.fixture(scope="session")  # noqa: ignore=F811
 @pytest.mark.usefixtures("app")
-def test_app(request, app):
+def test_app(request: pyramid.request.Request, app: Router) -> WebTestApp:
     testapp = WebTestApp(app)
     yield testapp