From 00c9b96068f7b7072b105c299ba12ed9f2df1b41 Mon Sep 17 00:00:00 2001 From: Saurabh Suchak <91744743+Saurabh-Suchak@users.noreply.github.com> Date: Sun, 16 Oct 2022 15:54:36 +0530 Subject: [PATCH 1/9] Improved Documentation for GCBM REST API (#144) * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * Update README.md * changed the endpoint name * Update app.py * Update README.md Co-authored-by: Harsh Mishra --- local/rest_api_gcbm/README.md | 103 +++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/local/rest_api_gcbm/README.md b/local/rest_api_gcbm/README.md index fae9e057..98170e3a 100644 --- a/local/rest_api_gcbm/README.md +++ b/local/rest_api_gcbm/README.md @@ -1,38 +1,89 @@ -# FLINT.Cloud - -### GCBM local-run REST API Setup - -Run the GCBM local-run container by pushing the following command: +# GCBM REST API Setup + +The simulation is currently supported on Linux/macOS. To run the GCBM simulation locally, execute the following steps: + +1. Clone the `FLINT.Cloud` repository using the command: + + ```bash + git clone https://github.com/moja-global/FLINT.Cloud.git + ``` + +2. Navigate to the `rest_api_gcbm` directory: + ```bash + cd FLINT.Cloud/local/rest_api_gcbm + ``` -```bash -docker-compose up -``` +3. Build the docker image with the image name `gcbm-api`: + ```bash + docker build --build-arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 -t gcbm-api . + ``` -Navigate to http://localhost:8080/ in the browser. You can stop the running container by pushing the following command: +4. Create a container using the `gcbm-api` image and start it: + ```bash + docker run --rm -p 8080:8080 gcbm-api + ``` -```bash -docker-compose down -``` +Navigate to http://localhost:8080/ in the browser to test the various endpoints available. -Currently the REST API has the following endpoints available for access:- +## Endpoints + +The GCBM REST API has the following endpoints available for access: | Endpoint | Functionality | Method | | :----------------: | :----------------: | :----------------: | -| **\help\all** | This endpoint produces a help message with information on all options for moja.CLI | `GET` -| **\help\arg** | This endpoint produces a help message with information on option arg for moja.CLI. | `GET` -| **\gcbm\new** | This endpoint creates a new directory to store input and output files of the simulation. Parameter to be passed in the body is the title of the new simulation or default value simulation will be used. | `POST` | -| **\gcbm\upload** | This endpoint is used to upload the input files: config_files, input and database. Remember to upload the files in the body from the GCBM folder which is available in the root of this repository for setup and use. A directory /input will be created to store the specified files. | `POST` | -| **\gcbm\dynamic** | This endpoint runs simulation in a thread and generates an output zip file in /output directory. The process may take a while. Parameter is the title of the simulation. | `POST` | -| **\gcbm\status** | This endpoint is employed to check the status of the simulation. It sends a message 'Output is ready to download' to notify that the output zip file is generated. Parameter is the title of the simulation. | `POST` -| **\gcbm\download** | This endpoint is used to download the output zip file. Parameter is the title of the simulation. | `POST` -| **\gcbm\list** | This endpoint retrieves the complete list of simulations that are created using /new. | `GET` +| **/help/all** | This endpoint produces a help message with information on all options for moja.CLI | `GET` +| **/help/arg** | This endpoint produces a help message with information on option arg for moja.CLI. | `GET` +| **/gcbm/new** | This endpoint creates a new directory to store input and output files of the simulation. Parameter to be passed in the body is the title of the new simulation or default value simulation will be used. | `POST` | +| **/gcbm/upload** | This endpoint is used to upload the input files: config_files, input and database. Remember to upload the files in the body from the GCBM folder which is available in the root of this repository for setup and use. A directory /input will be created to store the specified files. | `POST` | +| **/gcbm/run** | This endpoint runs simulation in a thread and generates an output zip file in /output directory. The process may take a while. Parameter is the title of the simulation. | `POST` | +| **/gcbm/status** | This endpoint is employed to check the status of the simulation. It sends a message 'Output is ready to download' to notify that the output zip file is generated. Parameter is the title of the simulation. | `POST` +| **/gcbm/download** | This endpoint is used to download the output zip file. Parameter is the title of the simulation. | `POST` +| **/gcbm/list** | This endpoint retrieves the complete list of simulations that are created using /new. | `GET` + +The inputs are contained in `GCBM_New_Demo_Run.zip`, present in the root of the directory. This file must be unzipped for further usage. Once the container is up and running, the following methods can be used to interact with the endpoints: + +1. A sample collection is available [in our Postaman collection](https://github.com/nynaalekhya/FLINT.Cloud/blob/local-gcbm-run/rest_local_run/local_run.postman_collection). Import the collection to Postman and run the endpoints. -The inputs are contained in `GCBM_Demo_Run.zip`, present in the root of the directory. This file must be unzipped for further usage. +2. The endpoints can be interacted with using `cURL`. `cURL` is used in command lines or scripts to transfer data. Find out more about curl [on the official website](https://curl.se/). Commands using `cURL` can be found [in our documentation](curl.md) -

Once the container is up and running, the following methods can be used to interact with the endpoints

+## Running a GCBM simulation -1. A sample Postman collection is available [here](https://github.com/nynaalekhya/FLINT.Cloud/blob/local-gcbm-run/rest_local_run/local_run.postman_collection). Import the collection to Postman and run the endpoints. +To run the endpoints and start a new GCBM simulation : + +1. Unzip `GCBM_New_Demo_Run` and open a terminal inside the folder and run the following curl commands: + +2. Create a new simulation (the default title of the simulation is `simulation`; Here the title `run4` is used): + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/new + ```` -2. The endpoints can be interacted with using `cURL`. -curl is used in command lines or scripts to transfer data. Find out more about curl [here](https://curl.se/). Commands using cURL can be found [here](curl.md) +3. Upload the input files: + ```bash + curl -F disturbances='@disturbances/disturbances_2011_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2012_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2013_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2014_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2015_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2016_moja.tiff' \ + -F disturbances='@disturbances/disturbances_2018_moja.tiff' \ + -F classifiers='@classifiers/Classifier1_moja.tiff' \ + -F classifiers='@classifiers/Classifier2_moja.tiff' \ + -F db='@db/gcbm_input.db' \ + -F miscellaneous='@miscellaneous/initial_age_moja.tiff' \ + -F miscellaneous='@miscellaneous/mean_annual_temperature_moja.tiff' \ + -F title="run4" \ + http://localhost:8080/gcbm/upload + ``` +4. To start the simulation, run: + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/run + ``` + + It should take around 35-40 minutes to finish running and even less depending on your local machine specifications. The end message should be `SQLite insert complete`. The message would be printed on the terminal used to execute Docker commands. + + +5. To download the simulation outputs, run the following curl command 3-5 minutes after the simulation has finished running: + ```bash + curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip + ``` From b711bc677d28a06646ec02c7770e831a06ae01f1 Mon Sep 17 00:00:00 2001 From: temitayo Date: Tue, 18 Oct 2022 14:11:20 +0100 Subject: [PATCH 2/9] fix: add a check to see if simulation output exists (#199) Signed-off-by: olalekan temitayo Signed-off-by: olalekan temitayo --- local/rest_api_gcbm/app.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/local/rest_api_gcbm/app.py b/local/rest_api_gcbm/app.py index f19a03ad..1feed739 100644 --- a/local/rest_api_gcbm/app.py +++ b/local/rest_api_gcbm/app.py @@ -462,9 +462,24 @@ def gcbm_download(): title = request.form.get("title") or "simulation" # Sanitize title title = "".join(c for c in title if c.isalnum()) + + output_dir = f"{os.getcwd()}/output/{title}.zip" + input_dir = f"{os.getcwd()}/input/{title}" + + # if the title has an input simulation and there is no output simulation then they should check the status. + if not os.path.exists(f"{output_dir}") and os.path.exists(f"{input_dir}"): + return { + "message": "You simulation is currently running, check the status via /gcbm/status" + } + + # if there is no input simulation and no output simulation then the simulation does not exist. + elif not os.path.exists(f"{output_dir}") and not os.path.exists(f"{input_dir}"): + return { + "message": "You don't have a simulation with this title kindly check the title and try again" + } + return send_file( - f"{os.getcwd()}/output/{title}.zip", - attachment_filename="{title}.zip", + f"{os.getcwd()}/output/{title}.zip", attachment_filename="{title}.zip", ) @@ -484,10 +499,13 @@ def gcbm_list_simulations(): for file in os.listdir(f"{os.getcwd()}/input"): list.append(file) - return { - "data": list, - "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.", - }, 200 + return ( + { + "data": list, + "message": "To create a new simulation, create a request at gcbm/new. To access the results of the existing simulations, create a request at gcbm/download.", + }, + 200, + ) @app.route("/gcbm/status", methods=["POST"]) From 7b06ef201ef3b4c5df90a6f4c1e6b89d980d4d96 Mon Sep 17 00:00:00 2001 From: Clinton Mekwunye Date: Sun, 23 Oct 2022 16:58:04 +0100 Subject: [PATCH 3/9] docs: document the `/gcbm/getConfig` endpoint (#201) Signed-off-by: Clinton Mekwunye Signed-off-by: Clinton Mekwunye --- local/rest_api_gcbm/curl.md | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/local/rest_api_gcbm/curl.md b/local/rest_api_gcbm/curl.md index e8b32653..1ef0532c 100644 --- a/local/rest_api_gcbm/curl.md +++ b/local/rest_api_gcbm/curl.md @@ -258,6 +258,90 @@ A file named `output.zip` will be obtained. This file contains the outputs gener curl -d "title=run4" -X POST http://localhost:8080/gcbm/download -L -o run4.zip ``` +### Get Config mean annual temperature moja file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="mean_annual_temperature_moja" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{ + "blockLatSize":0.1,"blockLonSize":0.1,"cellLatSize":0.00025, + "cellLonSize":0.00025,"layer_data":"Float32","layer_type":"GridLayer", + "nodata":32767.0,"tileLatSize":1.0,"tileLonSize":1.0 + } +} +``` + +### Get Config internal_variables file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="internal_variables" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{ + "Variables":{ + "LandUnitId":-1,"LocalDomainId":1,"age":0,"age_class":0,"blockIndex":0, + "cellIndex":0,"classifier_set":{},"current_land_class":"FL","historic_land_class":"FL", + "is_decaying":true,"is_forest":true,"landUnitArea":0,"landUnitBuildSuccess":true, + "localDomainId":0,"peatlandId":-1,"regen_delay":0,"run_delay":false,"run_moss":false, + "run_peatland":false,"simulateLandUnit":true, + "spatialLocationInfo":{ + "flintdata":{"library":"internal.flint","settings":{},"type":"SpatialLocationInfo"} + }, + "spinup_moss_only":false,"tileIndex":0,"unfccc_land_class":"UNFCCC_FL_R_FL" + } + } +} +``` + + +### Get Config localdomain file + +| **Method** | **Endpoint** | **Response code** | +| ---------- | ----------------- | ----------------- | +| POST | `/gcbm/getConfig` | 200 OK (Success) | + +``` +curl -F file_name="localdomain" \ + -F title="run4" \ + http://localhost:8080/gcbm/getConfig + +``` + +Expected response (Success): + +```json +{"data":{"Libraries":{"moja.modules.cbm":"external","moja.modules.gdal":"external"}, + "LocalDomain":{"end_date":"2021/01/01","landUnitBuildSuccess":"landUnitBuildSuccess", + "landscape":{"num_threads":4,"provider":"RasterTiled", + "tile_size_x":1.0,"tile_size_y":1.0, + "tiles":[{"index":12674,"x":-106,"y":55}], + "x_pixels":4000,"y_pixels":4000}, + "sequencer":"CBMSequencer","sequencer_library":"moja.modules.cbm", + "simulateLandUnit":"simulateLandUnit","start_date":"2010/01/01", + "timing":"annual","type":"spatial_tiled"}}} +``` + ### List all of the simulations | **Method** | **Endpoint** | **Response code** | From 76d829c81dc454fc63e46741bd64b1a57112bebc Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:13:44 +0100 Subject: [PATCH 4/9] GCBM Refactor idea Draft --- local/rest_api_skeleton/Endpoints/__init__.py | 0 .../Endpoints/gcbm_endpoints.py | 125 +++ local/rest_api_skeleton/Helpers/__init__.py | 0 .../rest_api_skeleton/Helpers/for_requests.py | 26 + local/rest_api_skeleton/Helpers/preprocess.py | 277 ++++++ local/rest_api_skeleton/__init__.py | 0 local/rest_api_skeleton/app.py | 16 + local/rest_api_skeleton/models/__init__.py | 0 local/rest_api_skeleton/templates/__init__.py | 0 .../templates/gcbm_config.cfg | 7 + .../templates/internal_variables.json | 34 + .../templates/localdomain.json | 31 + .../rest_api_skeleton/templates/logging.conf | 17 + .../templates/modules_cbm.json | 59 ++ .../templates/modules_output.json | 816 ++++++++++++++++++ .../templates/pools_cbm.json | 34 + .../templates/provider_config.json | 55 ++ local/rest_api_skeleton/templates/spinup.json | 56 ++ .../templates/variables.json | 257 ++++++ 19 files changed, 1810 insertions(+) create mode 100644 local/rest_api_skeleton/Endpoints/__init__.py create mode 100644 local/rest_api_skeleton/Endpoints/gcbm_endpoints.py create mode 100644 local/rest_api_skeleton/Helpers/__init__.py create mode 100644 local/rest_api_skeleton/Helpers/for_requests.py create mode 100644 local/rest_api_skeleton/Helpers/preprocess.py create mode 100644 local/rest_api_skeleton/__init__.py create mode 100644 local/rest_api_skeleton/app.py create mode 100644 local/rest_api_skeleton/models/__init__.py create mode 100644 local/rest_api_skeleton/templates/__init__.py create mode 100644 local/rest_api_skeleton/templates/gcbm_config.cfg create mode 100644 local/rest_api_skeleton/templates/internal_variables.json create mode 100644 local/rest_api_skeleton/templates/localdomain.json create mode 100644 local/rest_api_skeleton/templates/logging.conf create mode 100644 local/rest_api_skeleton/templates/modules_cbm.json create mode 100644 local/rest_api_skeleton/templates/modules_output.json create mode 100644 local/rest_api_skeleton/templates/pools_cbm.json create mode 100644 local/rest_api_skeleton/templates/provider_config.json create mode 100644 local/rest_api_skeleton/templates/spinup.json create mode 100644 local/rest_api_skeleton/templates/variables.json diff --git a/local/rest_api_skeleton/Endpoints/__init__.py b/local/rest_api_skeleton/Endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py new file mode 100644 index 00000000..46b865a0 --- /dev/null +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -0,0 +1,125 @@ +from flask_restful import Resource +import os +from threading import Thread +from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig +from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query + +class disturbance(Resource): + def post(self): + title = title_query().get("title") or "simulation" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + if not os.path.exists(f"{input_dir}/disturbances"): + os.makedirs(f"{input_dir}/disturbances") + + # store disturbances file in a new folder + disturbances = disturbance_query() + if not disturbances: + return {"error": "Missing disturbances file"}, 400 + for file in disturbances.get("disturbances"): + file.save(f"{input_dir}/disturbances/{file.filename}") + try: + disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb() + except Exception as e: + return e + disturb.flatten_directory("disturbances") + return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + +class classifier(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/classifiers"): + os.makedirs(f"{input_dir}/classifiers") + + # store disturbances file in a new folder + classifiers = classifier_query() + if not classifiers: + return {"error": "Missing classifiers file"}, 400 + + for file in classifiers.get("classifiers"): + file.save(f"{input_dir}/classifiers/{file.filename}") + try: + classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify() + except Exception as e: + return e + classify.flatten_directory("classifiers") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class Database(Resource): + def post(self): + pass + +class miscellaneous(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/miscellaneous"): + os.makedirs(f"{input_dir}/miscellaneous") + + # store miscellaneous file in a new folder + mis = miscellaneous_query() + if not mis: + return {"error": "Missing classifiers file"}, 400 + + for file in mis.get("miscellaneous"): + file.save(f"{input_dir}/miscellaneous/{file.filename}") + try: + miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel() + except Exception as e: + return e + miscel.flatten_directory("miscellaneous") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class title(Resource): + def post(self): + # Default title = simulation + title = title_query().get("title") or "simulation" + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + # input_dir = f"{title}" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + message = "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." + else: + message = "Simulation already exists. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." + + return {"data": message}, 200 + +class Run(Resource): + """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION + GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): + title = title_query().get("title") or "simulation" + + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + input_dir = f"{os.getcwd()}/input/{title}" + + + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread.start() + + return {"status": "Run started"}, 200 diff --git a/local/rest_api_skeleton/Helpers/__init__.py b/local/rest_api_skeleton/Helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py new file mode 100644 index 00000000..db53093d --- /dev/null +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -0,0 +1,26 @@ +from flask_restful import reqparse +from werkzeug.datastructures import FileStorage + +def title_query(): + query = reqparse.RequestParser() + query.add_argument("title", required=True, location="form" ) + return query.parse_args() + +def classifier_query(): + query = reqparse.RequestParser() + query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def disturbance_query(): + query = reqparse.RequestParser() + query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def miscellaneous_query(): + query = reqparse.RequestParser() + query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py new file mode 100644 index 00000000..157a0081 --- /dev/null +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -0,0 +1,277 @@ +import os, time, subprocess +import shutil +import json +import rasterio as rst + +def launch_run(title, input_dir): + s = time.time() + with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: + res = subprocess.Popen( + [ + "/opt/gcbm/moja.cli", + "--config_file", + "gcbm_config.cfg", + "--config_provider", + "provider_config.json", + ], + stdout=f, + cwd=f"{input_dir}", + ) + (_, err) = res.communicate() + + if not os.path.exists(f"{input_dir}/output"): + return "OK" + + # cut and paste output folder to app/output/simulation_name + shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}")) + shutil.make_archive( + f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}" + ) + shutil.rmtree((f"{input_dir}/output")) + e = time.time() + + response = { + "exitCode": res.returncode, + "execTime": e - s, + "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", + } + +class Configs: + + def __init__(self, input_dir) -> None: + self.input_dir = input_dir + self.Rasters = [] + self.Rastersm = [] + self.nodatam = [] + self.nodata = [] + self.cellLatSize = [] + self.cellLonSize = [] + self.paths = [] + self.lst = [] + self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + + def get_config_templates(self): + if not os.path.exists(f"{self.input_dir}/templates"): + shutil.copytree( + f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + ) + self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") + + def get_modules_cbm_config(self): + with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + data = json.load(modules_cbm_config) + disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) + data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances + json.dump(data, modules_cbm_config, indent=4) + modules_cbm_config.truncate() + + # for database input + def database_writes(self): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/db/"): + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + self.provider_config.seek(0) + + + def write_configs(self, config_type : str): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/{config_type}/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + self.lst.append(d) + if config_type == "disturbances" or config_type == "classifiers": + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): + for file in files: + fp = os.path.join(root, file) + self.Rasters.append(fp) + self.paths.append(fp) + for self.nd in self.Rasters: + img = rst.open(self.nd) + t = img.transform + x = t[0] + y = -t[4] + n = img.nodata + self.cellLatSize.append(x) + self.cellLonSize.append(y) + self.nodata.append(n) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + if result: + cellLat = x + cellLon = y + self.nd = n + blockLat = x * 400 + blockLon = y * 400 + tileLat = x * 400 + tileLon = y * 4000 + else: + print("Corrupt files") + + self.provider_config.seek(0) + new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} + data["Providers"]["RasterTiled"]["layers"] = self.lst + data["Providers"]["RasterTiled"].update(new_values) + + json.dump(data, self.provider_config, indent=4) + self.provider_config.truncate() + + self.dictionary = { + "layer_type": "GridLayer", + "layer_data": "Byte", + "nodata": self.nd, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + "blockLatSize": blockLat, + "blockLonSize": blockLon, + "cellLatSize": cellLat, + "cellLonSize": cellLon, + } + + self.study_area = { + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } + + def add_file_to_path(self, config_type): + for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for file in files: + fp = os.path.join(root, file) + self.paths.append(fp) + + def copy_directory(self): + for i in self.paths: + shutil.copy2(i, (f"{self.input_dir}")) + + def flatten_directory(self, config_type): + shutil.rmtree((f"{self.input_dir}/{config_type}/")) + +class DisturbanceConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def disturbances_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + for file in os.listdir(f"{self.input_dir}/disturbances/"): + study_area.append( + { + "name": file[:10], + "type": "DisturbanceLayer", + "tags": ["disturbance"], + } + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("disturbances") + self.disturbances_special() + self.add_file_to_path("disturbances") + self.copy_directory() + +class ClassifierConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def classifier_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + for file in os.listdir(f"{self.input_dir}/classifiers/"): + study_area.append( + {"name": file[:10], "type": "VectorLayer", "tags": ["classifier"]} + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("classifiers") + self.classifier_special() + self.add_file_to_path("classifiers") + self.copy_directory() + +class MiscellaneousConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def miscellaneous_special(self): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/miscellaneous/") +): + for file in files: + fp2 = os.path.join(root, file) + self.Rastersm.append(fp2) + + for i in self.Rastersm: + img = rst.open(i) + d = img.nodata + self.nodatam.append(d) + """this is an experimental thing""" + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["layer_type"] = "GridLayer" + if self.config_file == "mean_annual_temperature_moja.json": + self.dictionary["layer_data"] = "Float32" + else: + self.dictionary["layer_data"] = "Int16" + self.dictionary["nodata"] = 32767 + json.dump(self.dictionary, json_file, indent=4) + + # for study area + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + + for file in os.listdir(f"{self.input_dir}/miscellaneous/"): + study_area.append({"name": file[:10], "type": "VectorLayer"}) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("miscellaneous") + self.classifier_special() + self.add_file_to_path("miscellaneous") + self.copy_directory() + self.flatten_directory("miscellaneous") + + diff --git a/local/rest_api_skeleton/__init__.py b/local/rest_api_skeleton/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py new file mode 100644 index 00000000..c5502bc0 --- /dev/null +++ b/local/rest_api_skeleton/app.py @@ -0,0 +1,16 @@ +from flask import Flask +from flask_restful import Api +from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous + +app = Flask(__name__) +api = Api() + +api.add_resource(disturbance, "/gcbm/upload/disturbances") +api.add_resource(title, "/gcbm/create") +api.add_resource(classifier, "/gcbm/upload/classifies") +api.add_resource(miscellaneous, "/gcbm/upload/miscellaneous") +api.add_resource(Run, "/gcbm/run") + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/local/rest_api_skeleton/models/__init__.py b/local/rest_api_skeleton/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/__init__.py b/local/rest_api_skeleton/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/gcbm_config.cfg b/local/rest_api_skeleton/templates/gcbm_config.cfg new file mode 100644 index 00000000..e6640ac9 --- /dev/null +++ b/local/rest_api_skeleton/templates/gcbm_config.cfg @@ -0,0 +1,7 @@ +config=localdomain.json +config=pools_cbm.json +config=modules_cbm.json +config=modules_output.json +config=spinup.json +config=variables.json +config=internal_variables.json diff --git a/local/rest_api_skeleton/templates/internal_variables.json b/local/rest_api_skeleton/templates/internal_variables.json new file mode 100644 index 00000000..d9f71537 --- /dev/null +++ b/local/rest_api_skeleton/templates/internal_variables.json @@ -0,0 +1,34 @@ +{ + "Variables": { + "spatialLocationInfo": { + "flintdata": { + "type": "SpatialLocationInfo", + "library": "internal.flint", + "settings": {} + } + }, + "simulateLandUnit": true, + "is_decaying": true, + "spinup_moss_only": false, + "run_peatland": false, + "peatlandId": -1, + "is_forest": true, + "run_moss": false, + "run_delay": false, + "landUnitBuildSuccess": true, + "regen_delay": 0, + "age": 0, + "tileIndex": 0, + "blockIndex": 0, + "cellIndex": 0, + "LandUnitId": -1, + "landUnitArea": 0, + "classifier_set": {}, + "localDomainId": 0, + "LocalDomainId": 1, + "age_class": 0, + "historic_land_class": "FL", + "current_land_class": "FL", + "unfccc_land_class": "UNFCCC_FL_R_FL" + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/localdomain.json b/local/rest_api_skeleton/templates/localdomain.json new file mode 100644 index 00000000..5bb04f33 --- /dev/null +++ b/local/rest_api_skeleton/templates/localdomain.json @@ -0,0 +1,31 @@ +{ + "Libraries": { + "moja.modules.cbm": "external", + "moja.modules.gdal": "external" + }, + "LocalDomain": { + "start_date": "2010/01/01", + "end_date": "2021/01/01", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "simulateLandUnit": "simulateLandUnit", + "sequencer_library": "moja.modules.cbm", + "sequencer": "CBMSequencer", + "timing": "annual", + "type": "spatial_tiled", + "landscape": { + "provider": "RasterTiled", + "num_threads": 4, + "tiles": [ + { + "x": -106, + "y": 55, + "index": 12674 + } + ], + "x_pixels": 4000, + "y_pixels": 4000, + "tile_size_x": 1.0, + "tile_size_y": 1.0 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/logging.conf b/local/rest_api_skeleton/templates/logging.conf new file mode 100644 index 00000000..2da5fec5 --- /dev/null +++ b/local/rest_api_skeleton/templates/logging.conf @@ -0,0 +1,17 @@ +[Core] +DisableLogging=false + +[Sinks.console] +Destination=Console +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= info" + +[Sinks.file] +Destination=TextFile +FileName="output/simulation.log" +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= debug" diff --git a/local/rest_api_skeleton/templates/modules_cbm.json b/local/rest_api_skeleton/templates/modules_cbm.json new file mode 100644 index 00000000..0b237195 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_cbm.json @@ -0,0 +1,59 @@ +{ + "Modules": { + "CBMBuildLandUnitModule": { + "order": 1, + "library": "moja.modules.cbm" + }, + "CBMSequencer": { + "order": 2, + "library": "moja.modules.cbm" + }, + "CBMDisturbanceListener": { + "enabled": true, + "order": 3, + "library": "moja.modules.cbm", + "settings": { + "vars": [ + + ] + } + }, + "CBMDisturbanceEventModule": { + "enabled": true, + "order": 4, + "library": "moja.modules.cbm" + }, + "CBMTransitionRulesModule": { + "enabled": true, + "order": 5, + "library": "moja.modules.cbm" + }, + "CBMLandClassTransitionModule": { + "enabled": true, + "order": 6, + "library": "moja.modules.cbm" + }, + "CBMGrowthModule": { + "enabled": true, + "order": 7, + "library": "moja.modules.cbm" + }, + "CBMDecayModule": { + "enabled": true, + "order": 8, + "library": "moja.modules.cbm", + "settings": { + "extra_decay_removals": false + } + }, + "CBMAgeIndicators": { + "enabled": true, + "order": 9, + "library": "moja.modules.cbm" + }, + "TransactionManagerAfterSubmitModule": { + "order": 10, + "library": "internal.flint" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/modules_output.json b/local/rest_api_skeleton/templates/modules_output.json new file mode 100644 index 00000000..cad01b28 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_output.json @@ -0,0 +1,816 @@ +{ + "Modules": { + "WriteVariableGeotiff": { + "enabled": true, + "order": 11, + "library": "moja.modules.gdal", + "settings": { + "items": [ + { + "data_name": "Age", + "enabled": true, + "variable_data_type": "Int16", + "on_notification": "OutputStep", + "variable_name": "age" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther" + ], + "data_name": "AG_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Total_Ecosystem_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "data_name": "Total_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Dead_Organic_Matter_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "BelowGroundVeryFastSoil", + "BelowGroundSlowSoil" + ], + "data_name": "Soil_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + "data_name": "NPP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ], + "flux_source": "annual_process" + } + ], + "data_name": "NEP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "flux_source": "annual_process", + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "Decomp_Releases", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "NBP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + } + ], + "data_name": "Delta_Total_Ecosystem", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + } + ], + "data_name": "Delta_Total_Biomass", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO", + "Products" + ] + } + ], + "data_name": "Delta_Total_DOM", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ] + }, + "data_name": "Total_Biomass_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_DOM_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO2_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CH4" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CH4_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ] + } + ], + "data_name": "Ecosystem_Removals", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Bio_To_DOM_From_Disturbances", + "enabled": true, + "on_notification": "OutputStep" + } + ], + "output_path": "output" + } + }, + "CBMAggregatorLandUnitData": { + "enabled": true, + "order": 12, + "library": "moja.modules.cbm", + "settings": { + "reporting_classifier_set": "reporting_classifiers" + } + }, + "CBMAggregatorSQLiteWriter": { + "enabled": true, + "order": 13, + "library": "moja.modules.cbm", + "settings": { + "databasename": "output/simulation_output.db" + } + } + } +} diff --git a/local/rest_api_skeleton/templates/pools_cbm.json b/local/rest_api_skeleton/templates/pools_cbm.json new file mode 100644 index 00000000..35979d38 --- /dev/null +++ b/local/rest_api_skeleton/templates/pools_cbm.json @@ -0,0 +1,34 @@ +{ + "Pools": { + "AboveGroundFastSoil": 0.0, + "AboveGroundSlowSoil": 0.0, + "AboveGroundVeryFastSoil": 0.0, + "Atmosphere": 0.0, + "BelowGroundFastSoil": 0.0, + "BelowGroundSlowSoil": 0.0, + "BelowGroundVeryFastSoil": 0.0, + "BlackCarbon": 0.0, + "CH4": 0.0, + "CO": 0.0, + "CO2": 0.0, + "DissolvedOrganicCarbon": 0.0, + "HardwoodBranchSnag": 0.0, + "HardwoodCoarseRoots": 0.0, + "HardwoodFineRoots": 0.0, + "HardwoodFoliage": 0.0, + "HardwoodMerch": 0.0, + "HardwoodOther": 0.0, + "HardwoodStemSnag": 0.0, + "MediumSoil": 0.0, + "NO2": 0.0, + "Peat": 0.0, + "Products": 0.0, + "SoftwoodBranchSnag": 0.0, + "SoftwoodCoarseRoots": 0.0, + "SoftwoodFineRoots": 0.0, + "SoftwoodFoliage": 0.0, + "SoftwoodMerch": 0.0, + "SoftwoodOther": 0.0, + "SoftwoodStemSnag": 0.0 + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/provider_config.json b/local/rest_api_skeleton/templates/provider_config.json new file mode 100644 index 00000000..0888f9fc --- /dev/null +++ b/local/rest_api_skeleton/templates/provider_config.json @@ -0,0 +1,55 @@ +{ + "Providers": { + "SQLite": { + "path": "../input_database/gcbm_input.db", + "type": "SQLite" + }, + "RasterTiled": { + "layers": [ + { + "name": "disturbances_2011", + "layer_path": "disturbances_2011_moja.tiff", + "layer_prefix": "disturbances_2011_moja" + }, + { + "name": "disturbances_2012", + "layer_path": "disturbances_2012_moja.tiff", + "layer_prefix": "disturbances_2012_moja" + }, + { + "name": "disturbances_2013", + "layer_path": "disturbances_2013_moja.tiff", + "layer_prefix": "disturbances_2013_moja" + }, + { + "name": "disturbances_2014", + "layer_path": "disturbances_2014_moja.tiff", + "layer_prefix": "disturbances_2014_moja" + }, + { + "name": "disturbances_2015", + "layer_path": "disturbances_2015_moja.tiff", + "layer_prefix": "disturbances_2015_moja" + }, + { + "name": "disturbances_2016", + "layer_path": "disturbances_2016_moja.tiff", + "layer_prefix": "disturbances_2016_moja" + }, + { + "name": "disturbances_2018", + "layer_path": "disturbances_2018_moja.tiff", + "layer_prefix": "disturbances_2018_moja" + } + ], + "blockLonSize": 0.1, + "tileLatSize": 0.1, + "tileLonSize": 1.0, + "cellLatSize": 0.00025, + "cellLonSize": 0.00025, + "blockLatSize": 0.1, + "type": "RasterTiledGDAL", + "library": "moja.modules.gdal" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/spinup.json b/local/rest_api_skeleton/templates/spinup.json new file mode 100644 index 00000000..1365b141 --- /dev/null +++ b/local/rest_api_skeleton/templates/spinup.json @@ -0,0 +1,56 @@ +{ + "Spinup": { + "enabled": true, + "sequencer_library": "moja.modules.cbm", + "simulateLandUnit": "simulateLandUnit", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "sequencer": "CBMSpinupSequencer" + }, + "SpinupVariables": { + "delay": 0, + "minimum_rotation": 10, + "run_delay": false + }, + "Variables": { + "spinup_parameters": { + "transform": { + "queryString": "SELECT s.return_interval AS return_interval, s.max_rotations AS max_rotations, dt.name AS historic_disturbance_type, dt.name AS last_pass_disturbance_type, s.mean_annual_temperature AS mean_annual_temperature, 0 as delay FROM spinup_parameter s INNER JOIN disturbance_type dt ON s.historic_disturbance_type_id = dt.id INNER JOIN spatial_unit spu ON spu.spinup_parameter_id = s.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + } + }, + "SpinupModules": { + "CBMSpinupSequencer": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 1 + }, + "CBMBuildLandUnitModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 2 + }, + "CBMGrowthModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 3 + }, + "CBMDecayModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 4 + }, + "TransactionManagerAfterSubmitModule": { + "create_new": true, + "library": "internal.flint", + "order": 5 + }, + "CBMSpinupDisturbanceModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 6 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/variables.json b/local/rest_api_skeleton/templates/variables.json new file mode 100644 index 00000000..f2d8bc2f --- /dev/null +++ b/local/rest_api_skeleton/templates/variables.json @@ -0,0 +1,257 @@ +{ + "Variables": { + "enable_peatland": false, + "enable_moss": false, + "admin_boundary": "British Columbia", + "eco_boundary": "Taiga Plains", + "initial_age": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "initial_age" + } + }, + "initial_historic_land_class": "FL", + "initial_current_land_class": "FL", + "age_class_range": 20, + "age_maximum": 300, + "slow_ag_to_bg_mixing_rate": 0.006, + "disturbance_matrices": { + "transform": { + "queryString": "SELECT dm.id AS disturbance_matrix_id, source_pool.name as source_pool_name, dest_pool.name as dest_pool_name, dv.proportion FROM disturbance_matrix dm INNER JOIN disturbance_matrix_value dv ON dm.id = dv.disturbance_matrix_id INNER JOIN pool source_pool ON dv.source_pool_id = source_pool.id INNER JOIN pool dest_pool ON dv.sink_pool_id = dest_pool.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "softwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Softwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "reporting_classifiers": { + "transform": { + "allow_nulls": true, + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "classifier_set" + ] + } + }, + "land_class_transitions": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, lc.code AS land_class_transition, lc.is_forest, lc.years_to_permanent FROM disturbance_type dt INNER JOIN land_class lc ON dt.transition_land_class_id = lc.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rules": { + "transform": { + "queryString": "SELECT t.id AS id, age, regen_delay, description, tt.name AS reset_type FROM transition t INNER JOIN transition_type tt ON t.transition_type_id = tt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_matches": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "TransitionRuleTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "spatial_unit_id": { + "transform": { + "queryString": "SELECT spu.id FROM spatial_unit spu INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "hardwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Hardwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "turnover_rates": { + "transform": { + "queryString": "SELECT COALESCE(sw_turnover.foliage, 0) AS sw_foliage_turnover, COALESCE(hw_turnover.foliage, 0) AS hw_foliage_turnover, COALESCE(sw_turnover.stem, 0) AS sw_stem_turnover, COALESCE(hw_turnover.stem, 0) AS hw_stem_turnover, COALESCE(sw_turnover.branch, 0) AS sw_branch_turnover, COALESCE(hw_turnover.branch, 0) AS hw_branch_turnover, COALESCE(sw_turnover.branch_snag_split, 0) AS sw_other_to_branch_snag_split, COALESCE(hw_turnover.branch_snag_split, 0) AS hw_other_to_branch_snag_split, COALESCE(sw_turnover.stem_snag, 0) AS sw_stem_snag_turnover, COALESCE(hw_turnover.stem_snag, 0) AS hw_stem_snag_turnover, COALESCE(sw_turnover.branch_snag, 0) AS sw_branch_snag_turnover, COALESCE(hw_turnover.branch_snag, 0) AS hw_branch_snag_turnover, COALESCE(sw_turnover.coarse_ag_split, 0) AS sw_coarse_root_split, COALESCE(hw_turnover.coarse_ag_split, 0) AS hw_coarse_root_split, COALESCE(sw_turnover.coarse_root, 0) AS sw_coarse_root_turnover, COALESCE(hw_turnover.coarse_root, 0) AS hw_coarse_root_turnover, COALESCE(sw_turnover.fine_ag_split, 0) AS sw_fine_root_ag_split, COALESCE(hw_turnover.fine_ag_split, 0) AS hw_fine_root_ag_split, COALESCE(sw_turnover.fine_root, 0) AS sw_fine_root_turnover, COALESCE(hw_turnover.fine_root, 0) AS hw_fine_root_turnover FROM growth_curve gc LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Softwood' ORDER BY gcc.id LIMIT 1 ) AS sw_turnover ON gc.id = sw_turnover.growth_curve_id LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Hardwood' ORDER BY gcc.id LIMIT 1 ) AS hw_turnover ON gc.id = hw_turnover.growth_curve_id WHERE gc.id = {var:growth_curve_id}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbance_type_codes": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dt.code AS disturbance_type_code FROM disturbance_type dt", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_classifiers": { + "transform": { + "queryString": "SELECT t.id, c.name AS classifier_name, cv.value AS classifier_value FROM transition t INNER JOIN transition_classifier_value tcv ON t.id = tcv.transition_id INNER JOIN classifier_value cv ON tcv.classifier_value_id = cv.id INNER JOIN classifier c ON cv.classifier_id = c.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "initial_classifier_set": { + "transform": { + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "Classifier2", + "Classifier1" + ] + } + }, + "disturbance_matrix_associations": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dma.spatial_unit_id, dma.disturbance_matrix_id FROM disturbance_matrix_association dma INNER JOIN disturbance_type dt ON dma.disturbance_type_id = dt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "other_to_branch_snag_split": { + "transform": { + "queryString": "SELECT t.branch_snag_split AS slow_mixing_rate FROM eco_boundary e INNER JOIN turnover_parameter t ON e.turnover_parameter_id = t.id WHERE e.name LIKE {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "growth_curve_id": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "GrowthCurveTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "volume_to_biomass_parameters": { + "transform": { + "queryString": "SELECT ft.name AS forest_type, f.a as a, f.b as b, f.a_nonmerch as a_non_merch, f.b_nonmerch as b_non_merch, f.k_nonmerch as k_non_merch, f.cap_nonmerch as cap_non_merch, f.a_sap as a_sap, f.b_sap as b_sap, f.k_sap as k_sap, f.cap_sap as cap_sap, f.a1 as a1, f.a2 as a2, f.a3 as a3, f.b1 as b1, f.b2 as b2, f.b3 as b3, f.c1 as c1, f.c2 as c2, f.c3 as c3, f.min_volume as min_volume, f.max_volume as max_volume, f.low_stemwood_prop as low_stemwood_prop, f.high_stemwood_prop as high_stemwood_prop, f.low_stembark_prop as low_stembark_prop, f.high_stembark_prop as high_stembark_prop, f.low_branches_prop AS low_branches_prop, f.high_branches_prop as high_branches_prop, f.low_foliage_prop AS low_foliage_prop, f.high_foliage_prop AS high_foliage_prop, sp.sw_top_proportion AS softwood_top_prop, sp.sw_stump_proportion AS softwood_stump_prop, sp.hw_top_proportion AS hardwood_top_prop, sp.hw_stump_proportion AS hardwood_stump_prop, rp.hw_a AS hw_a, rp.hw_b AS hw_b, rp.sw_a AS sw_a, rp.frp_a AS frp_a, rp.frp_b AS frp_b, rp.frp_c AS frp_c FROM vol_to_bio_factor_association fa INNER JOIN vol_to_bio_factor f ON f.id = fa.vol_to_bio_factor_id INNER JOIN species s ON fa.species_id = s.id INNER JOIN growth_curve_component gcc ON s.id = gcc.species_id INNER JOIN forest_type ft ON s.forest_type_id = ft.id INNER JOIN spatial_unit spu ON fa.spatial_unit_id = spu.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN stump_parameter sp ON a.stump_parameter_id = sp.id INNER JOIN root_parameter rp ON rp.id = fa.root_parameter_id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND spu.id = {var:spatial_unit_id} ORDER BY gcc.id DESC", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "spu": { + "transform": { + "queryString": "select s.id AS spu_id from spatial_unit s inner join admin_boundary a on s.admin_boundary_id = a.id inner join eco_boundary e on s.eco_boundary_id = e.id where a.name like {var:admin_boundary} and e.name like {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "land_class_data": { + "transform": { + "queryString": "SELECT code AS land_class, is_forest, years_to_permanent FROM land_class lc", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "mean_annual_temperature": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "mean_annual_temperature" + } + }, + "decay_parameters": { + "transform": { + "queryString": "SELECT p.name AS pool, dp.base_decay_rate AS organic_matter_decay_rate, dp.prop_to_atmosphere AS prop_to_atmosphere, dp.q10 AS q10, dp.reference_temp AS reference_temp, dp.max_rate AS max_decay_rate_soft FROM decay_parameter dp INNER JOIN dom_pool dom ON dp.dom_pool_id = dom.id INNER JOIN pool p ON p.id = dom.pool_id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbances_2012": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2012" + } + }, + "disturbances_2011": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2011" + } + }, + "disturbances_2015": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2015" + } + }, + "disturbances_2014": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2014" + } + }, + "disturbances_2018": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2018" + } + }, + "disturbances_2016": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2016" + } + }, + "Classifier2": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier2" + } + }, + "Classifier1": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier1" + } + }, + "disturbances_2013": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2013" + } + } + } +} \ No newline at end of file From df1a6bd7310ca13036ab8c596f242eb5591c6caa Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:41:12 +0100 Subject: [PATCH 5/9] GCBM Refactor idea Draft --- .../Endpoints/gcbm_endpoints.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index 46b865a0..a5582bca 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,13 +1,24 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig -from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query +from Helpers.preprocess import ( + DisturbanceConfig, + launch_run, + ClassifierConfig, + MiscellaneousConfig, +) +from Helpers.for_requests import ( + title_query, + miscellaneous_query, + disturbance_query, + classifier_query, +) + class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -20,12 +31,17 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb = DisturbanceConfig( + input_dir, file.filename, disturbances.get("attributes") + ) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + disturb.flatten_directory("disturbances") + return { + "data": "Disturbances file uploaded succesfully. Proceed to the next step." + } + class classifier(Resource): def post(self): @@ -44,22 +60,28 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify = ClassifierConfig( + input_dir, file.filename, classifiers.get("attributes") + ) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class Database(Resource): def post(self): pass + class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -77,17 +99,22 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel = MiscellaneousConfig( + input_dir, file.filename, mis.get("attributes") + ) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class title(Resource): def post(self): @@ -105,9 +132,11 @@ def post(self): return {"data": message}, 200 + class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): title = title_query().get("title") or "simulation" @@ -115,11 +144,12 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) thread.start() return {"status": "Run started"}, 200 From 2ba12dc281909b3c4924d25859616cb7f7a4eccc Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:43:25 +0100 Subject: [PATCH 6/9] GCBM Refactor idea Draft --- .../rest_api_skeleton/Helpers/for_requests.py | 47 +++++-- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++++++------- local/rest_api_skeleton/app.py | 6 +- 3 files changed, 109 insertions(+), 61 deletions(-) diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index db53093d..9851264f 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,26 +1,47 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage + def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form" ) - return query.parse_args() + query.add_argument("title", required=True, location="form") + return query.parse_args() + def classifier_query(): query = reqparse.RequestParser() - query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "classifiers", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def disturbance_query(): query = reqparse.RequestParser() - query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "disturbances", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() - + query.add_argument( + "miscellaneous", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index 157a0081..a496b64a 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,7 +3,8 @@ import json import rasterio as rst -def launch_run(title, input_dir): + +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -36,32 +37,43 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } -class Configs: +class Configs: def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + self.provider_config = open( + f"{os.getcwd()}/templates/provider_config.json", "r+" + ) def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" ) - self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + with open( + f"{self.input_dir}/templates/modules_cbm.json", "r+" + ) as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [ + file.split(".")[0][:-5] + for file in os.listdir(f"{self.input_dir}/disturbances/") + ] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -70,11 +82,10 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} self.provider_config.seek(0) - - def write_configs(self, config_type : str): + def write_configs(self, config_type: str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -82,7 +93,7 @@ def write_configs(self, config_type : str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -91,15 +102,15 @@ def write_configs(self, config_type : str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -112,8 +123,15 @@ def write_configs(self, config_type : str): print("Corrupt files") self.provider_config.seek(0) - new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = { + "cellLonSize": cellLon, + "cellLatSize": cellLat, + "blockLonSize": blockLon, + "blockLatSize": blockLat, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + } + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -132,21 +150,23 @@ def write_configs(self, config_type : str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -157,15 +177,18 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - + + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -194,14 +217,17 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() + class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -225,16 +251,17 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() + class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") -): + ): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -244,9 +271,11 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -273,5 +302,3 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") - - diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index c5502bc0..cd256cc9 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True) From 68523b2a6fa57a8e5b5f8c385a87a5a304a72010 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:13:44 +0100 Subject: [PATCH 7/9] GCBM Refactor idea Draft Signed-off-by: Freeman --- local/rest_api_skeleton/Endpoints/__init__.py | 0 .../Endpoints/gcbm_endpoints.py | 125 +++ local/rest_api_skeleton/Helpers/__init__.py | 0 .../rest_api_skeleton/Helpers/for_requests.py | 26 + local/rest_api_skeleton/Helpers/preprocess.py | 277 ++++++ local/rest_api_skeleton/__init__.py | 0 local/rest_api_skeleton/app.py | 16 + local/rest_api_skeleton/models/__init__.py | 0 local/rest_api_skeleton/templates/__init__.py | 0 .../templates/gcbm_config.cfg | 7 + .../templates/internal_variables.json | 34 + .../templates/localdomain.json | 31 + .../rest_api_skeleton/templates/logging.conf | 17 + .../templates/modules_cbm.json | 59 ++ .../templates/modules_output.json | 816 ++++++++++++++++++ .../templates/pools_cbm.json | 34 + .../templates/provider_config.json | 55 ++ local/rest_api_skeleton/templates/spinup.json | 56 ++ .../templates/variables.json | 257 ++++++ 19 files changed, 1810 insertions(+) create mode 100644 local/rest_api_skeleton/Endpoints/__init__.py create mode 100644 local/rest_api_skeleton/Endpoints/gcbm_endpoints.py create mode 100644 local/rest_api_skeleton/Helpers/__init__.py create mode 100644 local/rest_api_skeleton/Helpers/for_requests.py create mode 100644 local/rest_api_skeleton/Helpers/preprocess.py create mode 100644 local/rest_api_skeleton/__init__.py create mode 100644 local/rest_api_skeleton/app.py create mode 100644 local/rest_api_skeleton/models/__init__.py create mode 100644 local/rest_api_skeleton/templates/__init__.py create mode 100644 local/rest_api_skeleton/templates/gcbm_config.cfg create mode 100644 local/rest_api_skeleton/templates/internal_variables.json create mode 100644 local/rest_api_skeleton/templates/localdomain.json create mode 100644 local/rest_api_skeleton/templates/logging.conf create mode 100644 local/rest_api_skeleton/templates/modules_cbm.json create mode 100644 local/rest_api_skeleton/templates/modules_output.json create mode 100644 local/rest_api_skeleton/templates/pools_cbm.json create mode 100644 local/rest_api_skeleton/templates/provider_config.json create mode 100644 local/rest_api_skeleton/templates/spinup.json create mode 100644 local/rest_api_skeleton/templates/variables.json diff --git a/local/rest_api_skeleton/Endpoints/__init__.py b/local/rest_api_skeleton/Endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py new file mode 100644 index 00000000..46b865a0 --- /dev/null +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -0,0 +1,125 @@ +from flask_restful import Resource +import os +from threading import Thread +from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig +from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query + +class disturbance(Resource): + def post(self): + title = title_query().get("title") or "simulation" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + if not os.path.exists(f"{input_dir}/disturbances"): + os.makedirs(f"{input_dir}/disturbances") + + # store disturbances file in a new folder + disturbances = disturbance_query() + if not disturbances: + return {"error": "Missing disturbances file"}, 400 + for file in disturbances.get("disturbances"): + file.save(f"{input_dir}/disturbances/{file.filename}") + try: + disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb() + except Exception as e: + return e + disturb.flatten_directory("disturbances") + return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + +class classifier(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/classifiers"): + os.makedirs(f"{input_dir}/classifiers") + + # store disturbances file in a new folder + classifiers = classifier_query() + if not classifiers: + return {"error": "Missing classifiers file"}, 400 + + for file in classifiers.get("classifiers"): + file.save(f"{input_dir}/classifiers/{file.filename}") + try: + classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify() + except Exception as e: + return e + classify.flatten_directory("classifiers") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class Database(Resource): + def post(self): + pass + +class miscellaneous(Resource): + def post(self): + # Get the title from the payload + title = title_query().get("title") or "simulation" + + # Check for project directory else create one + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + # input files follow a strict structure + if not os.path.exists(f"{input_dir}/miscellaneous"): + os.makedirs(f"{input_dir}/miscellaneous") + + # store miscellaneous file in a new folder + mis = miscellaneous_query() + if not mis: + return {"error": "Missing classifiers file"}, 400 + + for file in mis.get("miscellaneous"): + file.save(f"{input_dir}/miscellaneous/{file.filename}") + try: + miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel() + except Exception as e: + return e + miscel.flatten_directory("miscellaneous") + return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + +class title(Resource): + def post(self): + # Default title = simulation + title = title_query().get("title") or "simulation" + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + # input_dir = f"{title}" + input_dir = f"{os.getcwd()}/input/{title}" + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + message = "New simulation started. Please move on to the next stage for uploading files at /gcbm/upload." + else: + message = "Simulation already exists. Please check the list of simulations present before proceeding with a new simulation at gcbm/list. You may also download the input and output files for this simulation at gcbm/download sending parameter title in the body." + + return {"data": message}, 200 + +class Run(Resource): + """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION + GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): + title = title_query().get("title") or "simulation" + + # Sanitize title + title = "".join(c for c in title if c.isalnum()) + input_dir = f"{os.getcwd()}/input/{title}" + + + if not os.path.exists(f"{input_dir}"): + os.makedirs(f"{input_dir}") + + thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread.start() + + return {"status": "Run started"}, 200 diff --git a/local/rest_api_skeleton/Helpers/__init__.py b/local/rest_api_skeleton/Helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py new file mode 100644 index 00000000..db53093d --- /dev/null +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -0,0 +1,26 @@ +from flask_restful import reqparse +from werkzeug.datastructures import FileStorage + +def title_query(): + query = reqparse.RequestParser() + query.add_argument("title", required=True, location="form" ) + return query.parse_args() + +def classifier_query(): + query = reqparse.RequestParser() + query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def disturbance_query(): + query = reqparse.RequestParser() + query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + +def miscellaneous_query(): + query = reqparse.RequestParser() + query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) + query.add_argument("attributes", location= "form") + return query.parse_args() + diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py new file mode 100644 index 00000000..157a0081 --- /dev/null +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -0,0 +1,277 @@ +import os, time, subprocess +import shutil +import json +import rasterio as rst + +def launch_run(title, input_dir): + s = time.time() + with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: + res = subprocess.Popen( + [ + "/opt/gcbm/moja.cli", + "--config_file", + "gcbm_config.cfg", + "--config_provider", + "provider_config.json", + ], + stdout=f, + cwd=f"{input_dir}", + ) + (_, err) = res.communicate() + + if not os.path.exists(f"{input_dir}/output"): + return "OK" + + # cut and paste output folder to app/output/simulation_name + shutil.copytree(f"{input_dir}/output", (f"{os.getcwd()}/output/{title}")) + shutil.make_archive( + f"{os.getcwd()}/output/{title}", "zip", f"{os.getcwd()}/output/{title}" + ) + shutil.rmtree((f"{input_dir}/output")) + e = time.time() + + response = { + "exitCode": res.returncode, + "execTime": e - s, + "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", + } + +class Configs: + + def __init__(self, input_dir) -> None: + self.input_dir = input_dir + self.Rasters = [] + self.Rastersm = [] + self.nodatam = [] + self.nodata = [] + self.cellLatSize = [] + self.cellLonSize = [] + self.paths = [] + self.lst = [] + self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + + def get_config_templates(self): + if not os.path.exists(f"{self.input_dir}/templates"): + shutil.copytree( + f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + ) + self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") + + def get_modules_cbm_config(self): + with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + data = json.load(modules_cbm_config) + disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) + data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances + json.dump(data, modules_cbm_config, indent=4) + modules_cbm_config.truncate() + + # for database input + def database_writes(self): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/db/"): + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + self.provider_config.seek(0) + + + def write_configs(self, config_type : str): + data = json.load(self.provider_config) + for file in os.listdir(f"{self.input_dir}/{config_type}/"): + d = dict() + d["name"] = file[:-10] + d["layer_path"] = file + d["layer_prefix"] = file[:-5] + self.lst.append(d) + if config_type == "disturbances" or config_type == "classifiers": + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): + for file in files: + fp = os.path.join(root, file) + self.Rasters.append(fp) + self.paths.append(fp) + for self.nd in self.Rasters: + img = rst.open(self.nd) + t = img.transform + x = t[0] + y = -t[4] + n = img.nodata + self.cellLatSize.append(x) + self.cellLonSize.append(y) + self.nodata.append(n) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + if result: + cellLat = x + cellLon = y + self.nd = n + blockLat = x * 400 + blockLon = y * 400 + tileLat = x * 400 + tileLon = y * 4000 + else: + print("Corrupt files") + + self.provider_config.seek(0) + new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} + data["Providers"]["RasterTiled"]["layers"] = self.lst + data["Providers"]["RasterTiled"].update(new_values) + + json.dump(data, self.provider_config, indent=4) + self.provider_config.truncate() + + self.dictionary = { + "layer_type": "GridLayer", + "layer_data": "Byte", + "nodata": self.nd, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + "blockLatSize": blockLat, + "blockLonSize": blockLon, + "cellLatSize": cellLat, + "cellLonSize": cellLon, + } + + self.study_area = { + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } + + def add_file_to_path(self, config_type): + for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for file in files: + fp = os.path.join(root, file) + self.paths.append(fp) + + def copy_directory(self): + for i in self.paths: + shutil.copy2(i, (f"{self.input_dir}")) + + def flatten_directory(self, config_type): + shutil.rmtree((f"{self.input_dir}/{config_type}/")) + +class DisturbanceConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def disturbances_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + for file in os.listdir(f"{self.input_dir}/disturbances/"): + study_area.append( + { + "name": file[:10], + "type": "DisturbanceLayer", + "tags": ["disturbance"], + } + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("disturbances") + self.disturbances_special() + self.add_file_to_path("disturbances") + self.copy_directory() + +class ClassifierConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def classifier_special(self): + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["attributes"] = self.attribute + json.dump(self.dictionary, json_file, indent=4) + + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + for file in os.listdir(f"{self.input_dir}/classifiers/"): + study_area.append( + {"name": file[:10], "type": "VectorLayer", "tags": ["classifier"]} + ) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("classifiers") + self.classifier_special() + self.add_file_to_path("classifiers") + self.copy_directory() + +class MiscellaneousConfig(Configs): + def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + super().__init__(input_dir) + self.config_file = config_file + self.attribute = attribute + + def miscellaneous_special(self): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/miscellaneous/") +): + for file in files: + fp2 = os.path.join(root, file) + self.Rastersm.append(fp2) + + for i in self.Rastersm: + img = rst.open(i) + d = img.nodata + self.nodatam.append(d) + """this is an experimental thing""" + with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + self.dictionary["layer_type"] = "GridLayer" + if self.config_file == "mean_annual_temperature_moja.json": + self.dictionary["layer_data"] = "Float32" + else: + self.dictionary["layer_data"] = "Int16" + self.dictionary["nodata"] = 32767 + json.dump(self.dictionary, json_file, indent=4) + + # for study area + with open( + f"{self.input_dir}/study_area.json", "w", encoding="utf" + ) as json_file: + study_area = [] + + for file in os.listdir(f"{self.input_dir}/miscellaneous/"): + study_area.append({"name": file[:10], "type": "VectorLayer"}) + + self.study_area["layers"] = study_area + json.dump(self.study_area, json_file, indent=4) + + def __call__(self): + self.get_config_templates() + self.get_modules_cbm_config() + self.write_configs("miscellaneous") + self.classifier_special() + self.add_file_to_path("miscellaneous") + self.copy_directory() + self.flatten_directory("miscellaneous") + + diff --git a/local/rest_api_skeleton/__init__.py b/local/rest_api_skeleton/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py new file mode 100644 index 00000000..c5502bc0 --- /dev/null +++ b/local/rest_api_skeleton/app.py @@ -0,0 +1,16 @@ +from flask import Flask +from flask_restful import Api +from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous + +app = Flask(__name__) +api = Api() + +api.add_resource(disturbance, "/gcbm/upload/disturbances") +api.add_resource(title, "/gcbm/create") +api.add_resource(classifier, "/gcbm/upload/classifies") +api.add_resource(miscellaneous, "/gcbm/upload/miscellaneous") +api.add_resource(Run, "/gcbm/run") + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/local/rest_api_skeleton/models/__init__.py b/local/rest_api_skeleton/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/__init__.py b/local/rest_api_skeleton/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local/rest_api_skeleton/templates/gcbm_config.cfg b/local/rest_api_skeleton/templates/gcbm_config.cfg new file mode 100644 index 00000000..e6640ac9 --- /dev/null +++ b/local/rest_api_skeleton/templates/gcbm_config.cfg @@ -0,0 +1,7 @@ +config=localdomain.json +config=pools_cbm.json +config=modules_cbm.json +config=modules_output.json +config=spinup.json +config=variables.json +config=internal_variables.json diff --git a/local/rest_api_skeleton/templates/internal_variables.json b/local/rest_api_skeleton/templates/internal_variables.json new file mode 100644 index 00000000..d9f71537 --- /dev/null +++ b/local/rest_api_skeleton/templates/internal_variables.json @@ -0,0 +1,34 @@ +{ + "Variables": { + "spatialLocationInfo": { + "flintdata": { + "type": "SpatialLocationInfo", + "library": "internal.flint", + "settings": {} + } + }, + "simulateLandUnit": true, + "is_decaying": true, + "spinup_moss_only": false, + "run_peatland": false, + "peatlandId": -1, + "is_forest": true, + "run_moss": false, + "run_delay": false, + "landUnitBuildSuccess": true, + "regen_delay": 0, + "age": 0, + "tileIndex": 0, + "blockIndex": 0, + "cellIndex": 0, + "LandUnitId": -1, + "landUnitArea": 0, + "classifier_set": {}, + "localDomainId": 0, + "LocalDomainId": 1, + "age_class": 0, + "historic_land_class": "FL", + "current_land_class": "FL", + "unfccc_land_class": "UNFCCC_FL_R_FL" + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/localdomain.json b/local/rest_api_skeleton/templates/localdomain.json new file mode 100644 index 00000000..5bb04f33 --- /dev/null +++ b/local/rest_api_skeleton/templates/localdomain.json @@ -0,0 +1,31 @@ +{ + "Libraries": { + "moja.modules.cbm": "external", + "moja.modules.gdal": "external" + }, + "LocalDomain": { + "start_date": "2010/01/01", + "end_date": "2021/01/01", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "simulateLandUnit": "simulateLandUnit", + "sequencer_library": "moja.modules.cbm", + "sequencer": "CBMSequencer", + "timing": "annual", + "type": "spatial_tiled", + "landscape": { + "provider": "RasterTiled", + "num_threads": 4, + "tiles": [ + { + "x": -106, + "y": 55, + "index": 12674 + } + ], + "x_pixels": 4000, + "y_pixels": 4000, + "tile_size_x": 1.0, + "tile_size_y": 1.0 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/logging.conf b/local/rest_api_skeleton/templates/logging.conf new file mode 100644 index 00000000..2da5fec5 --- /dev/null +++ b/local/rest_api_skeleton/templates/logging.conf @@ -0,0 +1,17 @@ +[Core] +DisableLogging=false + +[Sinks.console] +Destination=Console +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= info" + +[Sinks.file] +Destination=TextFile +FileName="output/simulation.log" +Asynchronous=false +AutoFlush=true +Format="<%TimeStamp%> (%Severity%) - %Message%" +Filter="%Severity% >= debug" diff --git a/local/rest_api_skeleton/templates/modules_cbm.json b/local/rest_api_skeleton/templates/modules_cbm.json new file mode 100644 index 00000000..0b237195 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_cbm.json @@ -0,0 +1,59 @@ +{ + "Modules": { + "CBMBuildLandUnitModule": { + "order": 1, + "library": "moja.modules.cbm" + }, + "CBMSequencer": { + "order": 2, + "library": "moja.modules.cbm" + }, + "CBMDisturbanceListener": { + "enabled": true, + "order": 3, + "library": "moja.modules.cbm", + "settings": { + "vars": [ + + ] + } + }, + "CBMDisturbanceEventModule": { + "enabled": true, + "order": 4, + "library": "moja.modules.cbm" + }, + "CBMTransitionRulesModule": { + "enabled": true, + "order": 5, + "library": "moja.modules.cbm" + }, + "CBMLandClassTransitionModule": { + "enabled": true, + "order": 6, + "library": "moja.modules.cbm" + }, + "CBMGrowthModule": { + "enabled": true, + "order": 7, + "library": "moja.modules.cbm" + }, + "CBMDecayModule": { + "enabled": true, + "order": 8, + "library": "moja.modules.cbm", + "settings": { + "extra_decay_removals": false + } + }, + "CBMAgeIndicators": { + "enabled": true, + "order": 9, + "library": "moja.modules.cbm" + }, + "TransactionManagerAfterSubmitModule": { + "order": 10, + "library": "internal.flint" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/modules_output.json b/local/rest_api_skeleton/templates/modules_output.json new file mode 100644 index 00000000..cad01b28 --- /dev/null +++ b/local/rest_api_skeleton/templates/modules_output.json @@ -0,0 +1,816 @@ +{ + "Modules": { + "WriteVariableGeotiff": { + "enabled": true, + "order": 11, + "library": "moja.modules.gdal", + "settings": { + "items": [ + { + "data_name": "Age", + "enabled": true, + "variable_data_type": "Int16", + "on_notification": "OutputStep", + "variable_name": "age" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther" + ], + "data_name": "AG_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Total_Ecosystem_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "data_name": "Total_Biomass_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "data_name": "Dead_Organic_Matter_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "pool_name": [ + "BelowGroundVeryFastSoil", + "BelowGroundSlowSoil" + ], + "data_name": "Soil_C", + "enabled": true, + "variable_data_type": "float", + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + "data_name": "NPP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ], + "flux_source": "annual_process" + } + ], + "data_name": "NEP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "flux_source": "annual_process", + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "Decomp_Releases", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + } + ], + "data_name": "NBP", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + } + ], + "data_name": "Delta_Total_Ecosystem", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "from": [ + "Atmosphere" + ] + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO" + ] + }, + { + "subtract": true, + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "Products" + ], + "flux_source": "disturbance" + }, + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "subtract": true, + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + } + ], + "data_name": "Delta_Total_Biomass", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "subtract": true, + "to": [ + "CO2", + "CH4", + "CO", + "Products" + ] + } + ], + "data_name": "Delta_Total_DOM", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ] + }, + "data_name": "Total_Biomass_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_DOM_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO2" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO2_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CO_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "to": [ + "CH4" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Total_CH4_Emissions", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": [ + { + "to": [ + "CO2", + "CH4", + "CO" + ], + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ], + "to": [ + "Products" + ] + } + ], + "data_name": "Ecosystem_Removals", + "enabled": true, + "on_notification": "OutputStep" + }, + { + "variable_data_type": "float", + "flux": { + "flux_source": "disturbance", + "from": [ + "SoftwoodMerch", + "SoftwoodFoliage", + "SoftwoodOther", + "SoftwoodCoarseRoots", + "SoftwoodFineRoots", + "HardwoodMerch", + "HardwoodFoliage", + "HardwoodOther", + "HardwoodCoarseRoots", + "HardwoodFineRoots" + ], + "to": [ + "AboveGroundVeryFastSoil", + "BelowGroundVeryFastSoil", + "AboveGroundFastSoil", + "BelowGroundFastSoil", + "MediumSoil", + "AboveGroundSlowSoil", + "BelowGroundSlowSoil", + "SoftwoodStemSnag", + "SoftwoodBranchSnag", + "HardwoodStemSnag", + "HardwoodBranchSnag" + ] + }, + "data_name": "Bio_To_DOM_From_Disturbances", + "enabled": true, + "on_notification": "OutputStep" + } + ], + "output_path": "output" + } + }, + "CBMAggregatorLandUnitData": { + "enabled": true, + "order": 12, + "library": "moja.modules.cbm", + "settings": { + "reporting_classifier_set": "reporting_classifiers" + } + }, + "CBMAggregatorSQLiteWriter": { + "enabled": true, + "order": 13, + "library": "moja.modules.cbm", + "settings": { + "databasename": "output/simulation_output.db" + } + } + } +} diff --git a/local/rest_api_skeleton/templates/pools_cbm.json b/local/rest_api_skeleton/templates/pools_cbm.json new file mode 100644 index 00000000..35979d38 --- /dev/null +++ b/local/rest_api_skeleton/templates/pools_cbm.json @@ -0,0 +1,34 @@ +{ + "Pools": { + "AboveGroundFastSoil": 0.0, + "AboveGroundSlowSoil": 0.0, + "AboveGroundVeryFastSoil": 0.0, + "Atmosphere": 0.0, + "BelowGroundFastSoil": 0.0, + "BelowGroundSlowSoil": 0.0, + "BelowGroundVeryFastSoil": 0.0, + "BlackCarbon": 0.0, + "CH4": 0.0, + "CO": 0.0, + "CO2": 0.0, + "DissolvedOrganicCarbon": 0.0, + "HardwoodBranchSnag": 0.0, + "HardwoodCoarseRoots": 0.0, + "HardwoodFineRoots": 0.0, + "HardwoodFoliage": 0.0, + "HardwoodMerch": 0.0, + "HardwoodOther": 0.0, + "HardwoodStemSnag": 0.0, + "MediumSoil": 0.0, + "NO2": 0.0, + "Peat": 0.0, + "Products": 0.0, + "SoftwoodBranchSnag": 0.0, + "SoftwoodCoarseRoots": 0.0, + "SoftwoodFineRoots": 0.0, + "SoftwoodFoliage": 0.0, + "SoftwoodMerch": 0.0, + "SoftwoodOther": 0.0, + "SoftwoodStemSnag": 0.0 + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/provider_config.json b/local/rest_api_skeleton/templates/provider_config.json new file mode 100644 index 00000000..0888f9fc --- /dev/null +++ b/local/rest_api_skeleton/templates/provider_config.json @@ -0,0 +1,55 @@ +{ + "Providers": { + "SQLite": { + "path": "../input_database/gcbm_input.db", + "type": "SQLite" + }, + "RasterTiled": { + "layers": [ + { + "name": "disturbances_2011", + "layer_path": "disturbances_2011_moja.tiff", + "layer_prefix": "disturbances_2011_moja" + }, + { + "name": "disturbances_2012", + "layer_path": "disturbances_2012_moja.tiff", + "layer_prefix": "disturbances_2012_moja" + }, + { + "name": "disturbances_2013", + "layer_path": "disturbances_2013_moja.tiff", + "layer_prefix": "disturbances_2013_moja" + }, + { + "name": "disturbances_2014", + "layer_path": "disturbances_2014_moja.tiff", + "layer_prefix": "disturbances_2014_moja" + }, + { + "name": "disturbances_2015", + "layer_path": "disturbances_2015_moja.tiff", + "layer_prefix": "disturbances_2015_moja" + }, + { + "name": "disturbances_2016", + "layer_path": "disturbances_2016_moja.tiff", + "layer_prefix": "disturbances_2016_moja" + }, + { + "name": "disturbances_2018", + "layer_path": "disturbances_2018_moja.tiff", + "layer_prefix": "disturbances_2018_moja" + } + ], + "blockLonSize": 0.1, + "tileLatSize": 0.1, + "tileLonSize": 1.0, + "cellLatSize": 0.00025, + "cellLonSize": 0.00025, + "blockLatSize": 0.1, + "type": "RasterTiledGDAL", + "library": "moja.modules.gdal" + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/spinup.json b/local/rest_api_skeleton/templates/spinup.json new file mode 100644 index 00000000..1365b141 --- /dev/null +++ b/local/rest_api_skeleton/templates/spinup.json @@ -0,0 +1,56 @@ +{ + "Spinup": { + "enabled": true, + "sequencer_library": "moja.modules.cbm", + "simulateLandUnit": "simulateLandUnit", + "landUnitBuildSuccess": "landUnitBuildSuccess", + "sequencer": "CBMSpinupSequencer" + }, + "SpinupVariables": { + "delay": 0, + "minimum_rotation": 10, + "run_delay": false + }, + "Variables": { + "spinup_parameters": { + "transform": { + "queryString": "SELECT s.return_interval AS return_interval, s.max_rotations AS max_rotations, dt.name AS historic_disturbance_type, dt.name AS last_pass_disturbance_type, s.mean_annual_temperature AS mean_annual_temperature, 0 as delay FROM spinup_parameter s INNER JOIN disturbance_type dt ON s.historic_disturbance_type_id = dt.id INNER JOIN spatial_unit spu ON spu.spinup_parameter_id = s.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + } + }, + "SpinupModules": { + "CBMSpinupSequencer": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 1 + }, + "CBMBuildLandUnitModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 2 + }, + "CBMGrowthModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 3 + }, + "CBMDecayModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 4 + }, + "TransactionManagerAfterSubmitModule": { + "create_new": true, + "library": "internal.flint", + "order": 5 + }, + "CBMSpinupDisturbanceModule": { + "create_new": true, + "library": "moja.modules.cbm", + "order": 6 + } + } +} \ No newline at end of file diff --git a/local/rest_api_skeleton/templates/variables.json b/local/rest_api_skeleton/templates/variables.json new file mode 100644 index 00000000..f2d8bc2f --- /dev/null +++ b/local/rest_api_skeleton/templates/variables.json @@ -0,0 +1,257 @@ +{ + "Variables": { + "enable_peatland": false, + "enable_moss": false, + "admin_boundary": "British Columbia", + "eco_boundary": "Taiga Plains", + "initial_age": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "initial_age" + } + }, + "initial_historic_land_class": "FL", + "initial_current_land_class": "FL", + "age_class_range": 20, + "age_maximum": 300, + "slow_ag_to_bg_mixing_rate": 0.006, + "disturbance_matrices": { + "transform": { + "queryString": "SELECT dm.id AS disturbance_matrix_id, source_pool.name as source_pool_name, dest_pool.name as dest_pool_name, dv.proportion FROM disturbance_matrix dm INNER JOIN disturbance_matrix_value dv ON dm.id = dv.disturbance_matrix_id INNER JOIN pool source_pool ON dv.source_pool_id = source_pool.id INNER JOIN pool dest_pool ON dv.sink_pool_id = dest_pool.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "softwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Softwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "reporting_classifiers": { + "transform": { + "allow_nulls": true, + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "classifier_set" + ] + } + }, + "land_class_transitions": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, lc.code AS land_class_transition, lc.is_forest, lc.years_to_permanent FROM disturbance_type dt INNER JOIN land_class lc ON dt.transition_land_class_id = lc.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rules": { + "transform": { + "queryString": "SELECT t.id AS id, age, regen_delay, description, tt.name AS reset_type FROM transition t INNER JOIN transition_type tt ON t.transition_type_id = tt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_matches": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "TransitionRuleTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "spatial_unit_id": { + "transform": { + "queryString": "SELECT spu.id FROM spatial_unit spu INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN eco_boundary e ON spu.eco_boundary_id = e.id WHERE a.name = {var:admin_boundary} AND e.name = {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "hardwood_yield_table": { + "transform": { + "queryString": "SELECT gcv.age AS age, SUM(gcv.merchantable_volume) AS merchantable_volume FROM (SELECT CASE WHEN gc.id IS NOT NULL THEN gc.id ELSE -1 END AS growth_curve_component_id FROM growth_curve_component gc INNER JOIN species s ON s.id = gc.species_id INNER JOIN forest_type ft ON ft.id = s.forest_type_id WHERE gc.growth_curve_id = {var:growth_curve_id} AND LOWER(ft.name) LIKE LOWER('Hardwood')) AS gc INNER JOIN growth_curve_component_value gcv ON gc.growth_curve_component_id = gcv.growth_curve_component_id GROUP BY gcv.age", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "turnover_rates": { + "transform": { + "queryString": "SELECT COALESCE(sw_turnover.foliage, 0) AS sw_foliage_turnover, COALESCE(hw_turnover.foliage, 0) AS hw_foliage_turnover, COALESCE(sw_turnover.stem, 0) AS sw_stem_turnover, COALESCE(hw_turnover.stem, 0) AS hw_stem_turnover, COALESCE(sw_turnover.branch, 0) AS sw_branch_turnover, COALESCE(hw_turnover.branch, 0) AS hw_branch_turnover, COALESCE(sw_turnover.branch_snag_split, 0) AS sw_other_to_branch_snag_split, COALESCE(hw_turnover.branch_snag_split, 0) AS hw_other_to_branch_snag_split, COALESCE(sw_turnover.stem_snag, 0) AS sw_stem_snag_turnover, COALESCE(hw_turnover.stem_snag, 0) AS hw_stem_snag_turnover, COALESCE(sw_turnover.branch_snag, 0) AS sw_branch_snag_turnover, COALESCE(hw_turnover.branch_snag, 0) AS hw_branch_snag_turnover, COALESCE(sw_turnover.coarse_ag_split, 0) AS sw_coarse_root_split, COALESCE(hw_turnover.coarse_ag_split, 0) AS hw_coarse_root_split, COALESCE(sw_turnover.coarse_root, 0) AS sw_coarse_root_turnover, COALESCE(hw_turnover.coarse_root, 0) AS hw_coarse_root_turnover, COALESCE(sw_turnover.fine_ag_split, 0) AS sw_fine_root_ag_split, COALESCE(hw_turnover.fine_ag_split, 0) AS hw_fine_root_ag_split, COALESCE(sw_turnover.fine_root, 0) AS sw_fine_root_turnover, COALESCE(hw_turnover.fine_root, 0) AS hw_fine_root_turnover FROM growth_curve gc LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Softwood' ORDER BY gcc.id LIMIT 1 ) AS sw_turnover ON gc.id = sw_turnover.growth_curve_id LEFT JOIN ( SELECT growth_curve_id, foliage, stem, branch, branch_snag_split, stem_snag, branch_snag, coarse_ag_split, coarse_root, fine_ag_split, fine_root FROM turnover_parameter_association tpa INNER JOIN eco_boundary e ON tpa.eco_boundary_id = e.id INNER JOIN genus g ON tpa.genus_id = g.id INNER JOIN species s ON s.genus_id = g.id INNER JOIN forest_type f ON s.forest_type_id = f.id INNER JOIN growth_curve_component gcc ON gcc.species_id = s.id INNER JOIN turnover_parameter t ON tpa.turnover_parameter_id = t.id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND e.name = {var:eco_boundary} AND f.name = 'Hardwood' ORDER BY gcc.id LIMIT 1 ) AS hw_turnover ON gc.id = hw_turnover.growth_curve_id WHERE gc.id = {var:growth_curve_id}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbance_type_codes": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dt.code AS disturbance_type_code FROM disturbance_type dt", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "transition_rule_classifiers": { + "transform": { + "queryString": "SELECT t.id, c.name AS classifier_name, cv.value AS classifier_value FROM transition t INNER JOIN transition_classifier_value tcv ON t.id = tcv.transition_id INNER JOIN classifier_value cv ON tcv.classifier_value_id = cv.id INNER JOIN classifier c ON cv.classifier_id = c.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "initial_classifier_set": { + "transform": { + "type": "CompositeTransform", + "library": "internal.flint", + "vars": [ + "Classifier2", + "Classifier1" + ] + } + }, + "disturbance_matrix_associations": { + "transform": { + "queryString": "SELECT dt.name AS disturbance_type, dma.spatial_unit_id, dma.disturbance_matrix_id FROM disturbance_matrix_association dma INNER JOIN disturbance_type dt ON dma.disturbance_type_id = dt.id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "other_to_branch_snag_split": { + "transform": { + "queryString": "SELECT t.branch_snag_split AS slow_mixing_rate FROM eco_boundary e INNER JOIN turnover_parameter t ON e.turnover_parameter_id = t.id WHERE e.name LIKE {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "growth_curve_id": { + "transform": { + "classifier_set_var": "classifier_set", + "type": "GrowthCurveTransform", + "library": "moja.modules.cbm", + "provider": "SQLite" + } + }, + "volume_to_biomass_parameters": { + "transform": { + "queryString": "SELECT ft.name AS forest_type, f.a as a, f.b as b, f.a_nonmerch as a_non_merch, f.b_nonmerch as b_non_merch, f.k_nonmerch as k_non_merch, f.cap_nonmerch as cap_non_merch, f.a_sap as a_sap, f.b_sap as b_sap, f.k_sap as k_sap, f.cap_sap as cap_sap, f.a1 as a1, f.a2 as a2, f.a3 as a3, f.b1 as b1, f.b2 as b2, f.b3 as b3, f.c1 as c1, f.c2 as c2, f.c3 as c3, f.min_volume as min_volume, f.max_volume as max_volume, f.low_stemwood_prop as low_stemwood_prop, f.high_stemwood_prop as high_stemwood_prop, f.low_stembark_prop as low_stembark_prop, f.high_stembark_prop as high_stembark_prop, f.low_branches_prop AS low_branches_prop, f.high_branches_prop as high_branches_prop, f.low_foliage_prop AS low_foliage_prop, f.high_foliage_prop AS high_foliage_prop, sp.sw_top_proportion AS softwood_top_prop, sp.sw_stump_proportion AS softwood_stump_prop, sp.hw_top_proportion AS hardwood_top_prop, sp.hw_stump_proportion AS hardwood_stump_prop, rp.hw_a AS hw_a, rp.hw_b AS hw_b, rp.sw_a AS sw_a, rp.frp_a AS frp_a, rp.frp_b AS frp_b, rp.frp_c AS frp_c FROM vol_to_bio_factor_association fa INNER JOIN vol_to_bio_factor f ON f.id = fa.vol_to_bio_factor_id INNER JOIN species s ON fa.species_id = s.id INNER JOIN growth_curve_component gcc ON s.id = gcc.species_id INNER JOIN forest_type ft ON s.forest_type_id = ft.id INNER JOIN spatial_unit spu ON fa.spatial_unit_id = spu.id INNER JOIN admin_boundary a ON spu.admin_boundary_id = a.id INNER JOIN stump_parameter sp ON a.stump_parameter_id = sp.id INNER JOIN root_parameter rp ON rp.id = fa.root_parameter_id WHERE gcc.growth_curve_id = {var:growth_curve_id} AND spu.id = {var:spatial_unit_id} ORDER BY gcc.id DESC", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "spu": { + "transform": { + "queryString": "select s.id AS spu_id from spatial_unit s inner join admin_boundary a on s.admin_boundary_id = a.id inner join eco_boundary e on s.eco_boundary_id = e.id where a.name like {var:admin_boundary} and e.name like {var:eco_boundary}", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "land_class_data": { + "transform": { + "queryString": "SELECT code AS land_class, is_forest, years_to_permanent FROM land_class lc", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "mean_annual_temperature": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "mean_annual_temperature" + } + }, + "decay_parameters": { + "transform": { + "queryString": "SELECT p.name AS pool, dp.base_decay_rate AS organic_matter_decay_rate, dp.prop_to_atmosphere AS prop_to_atmosphere, dp.q10 AS q10, dp.reference_temp AS reference_temp, dp.max_rate AS max_decay_rate_soft FROM decay_parameter dp INNER JOIN dom_pool dom ON dp.dom_pool_id = dom.id INNER JOIN pool p ON p.id = dom.pool_id", + "type": "SQLQueryTransform", + "library": "internal.flint", + "provider": "SQLite" + } + }, + "disturbances_2012": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2012" + } + }, + "disturbances_2011": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2011" + } + }, + "disturbances_2015": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2015" + } + }, + "disturbances_2014": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2014" + } + }, + "disturbances_2018": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2018" + } + }, + "disturbances_2016": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2016" + } + }, + "Classifier2": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier2" + } + }, + "Classifier1": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "Classifier1" + } + }, + "disturbances_2013": { + "transform": { + "library": "internal.flint", + "type": "LocationIdxFromFlintDataTransform", + "provider": "RasterTiled", + "data_id": "disturbances_2013" + } + } + } +} \ No newline at end of file From dd4077026610f525e28a9ac009f73667a0e7fa3d Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:41:12 +0100 Subject: [PATCH 8/9] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../Endpoints/gcbm_endpoints.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py index 46b865a0..a5582bca 100644 --- a/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py +++ b/local/rest_api_skeleton/Endpoints/gcbm_endpoints.py @@ -1,13 +1,24 @@ -from flask_restful import Resource +from flask_restful import Resource import os from threading import Thread -from Helpers.preprocess import DisturbanceConfig, launch_run, ClassifierConfig, MiscellaneousConfig -from Helpers.for_requests import title_query, miscellaneous_query, disturbance_query, classifier_query +from Helpers.preprocess import ( + DisturbanceConfig, + launch_run, + ClassifierConfig, + MiscellaneousConfig, +) +from Helpers.for_requests import ( + title_query, + miscellaneous_query, + disturbance_query, + classifier_query, +) + class disturbance(Resource): def post(self): title = title_query().get("title") or "simulation" - input_dir = f"{os.getcwd()}/input/{title}" + input_dir = f"{os.getcwd()}/input/{title}" if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") if not os.path.exists(f"{input_dir}/disturbances"): @@ -20,12 +31,17 @@ def post(self): for file in disturbances.get("disturbances"): file.save(f"{input_dir}/disturbances/{file.filename}") try: - disturb = DisturbanceConfig(input_dir, file.filename, disturbances.get("attributes")) + disturb = DisturbanceConfig( + input_dir, file.filename, disturbances.get("attributes") + ) disturb() - except Exception as e: + except Exception as e: return e - disturb.flatten_directory("disturbances") - return {"data": "Disturbances file uploaded succesfully. Proceed to the next step."} + disturb.flatten_directory("disturbances") + return { + "data": "Disturbances file uploaded succesfully. Proceed to the next step." + } + class classifier(Resource): def post(self): @@ -44,22 +60,28 @@ def post(self): # store disturbances file in a new folder classifiers = classifier_query() if not classifiers: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in classifiers.get("classifiers"): file.save(f"{input_dir}/classifiers/{file.filename}") try: - classify = ClassifierConfig(input_dir, file.filename, classifiers.get("attributes")) + classify = ClassifierConfig( + input_dir, file.filename, classifiers.get("attributes") + ) classify() except Exception as e: return e classify.flatten_directory("classifiers") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class Database(Resource): def post(self): pass + class miscellaneous(Resource): def post(self): # Get the title from the payload @@ -77,17 +99,22 @@ def post(self): # store miscellaneous file in a new folder mis = miscellaneous_query() if not mis: - return {"error": "Missing classifiers file"}, 400 + return {"error": "Missing classifiers file"}, 400 for file in mis.get("miscellaneous"): file.save(f"{input_dir}/miscellaneous/{file.filename}") try: - miscel = MiscellaneousConfig(input_dir, file.filename, mis.get("attributes")) + miscel = MiscellaneousConfig( + input_dir, file.filename, mis.get("attributes") + ) miscel() except Exception as e: return e miscel.flatten_directory("miscellaneous") - return {"data": "Classifiers file uploaded succesfully. Proceed to the next step."} + return { + "data": "Classifiers file uploaded succesfully. Proceed to the next step." + } + class title(Resource): def post(self): @@ -105,9 +132,11 @@ def post(self): return {"data": message}, 200 + class Run(Resource): """THIS ENDPOINT WILL BE ABLE TO RUN A SIMULATION GOAL IS TO EQUIP IT TO BE ABLE TO RUN MORE THAN ONE SIMULATIONS AT A TIME""" + def post(self): title = title_query().get("title") or "simulation" @@ -115,11 +144,12 @@ def post(self): title = "".join(c for c in title if c.isalnum()) input_dir = f"{os.getcwd()}/input/{title}" - if not os.path.exists(f"{input_dir}"): os.makedirs(f"{input_dir}") - thread = Thread(target=launch_run, kwargs={"title": title, "input_dir": input_dir}) + thread = Thread( + target=launch_run, kwargs={"title": title, "input_dir": input_dir} + ) thread.start() return {"status": "Run started"}, 200 From 44402d9012a7f22333083b1c57f96d659a7f4f10 Mon Sep 17 00:00:00 2001 From: Freeman Date: Fri, 28 Oct 2022 00:43:25 +0100 Subject: [PATCH 9/9] GCBM Refactor idea Draft Signed-off-by: Freeman --- .../rest_api_skeleton/Helpers/for_requests.py | 47 +++++-- local/rest_api_skeleton/Helpers/preprocess.py | 117 +++++++++++------- local/rest_api_skeleton/app.py | 6 +- 3 files changed, 109 insertions(+), 61 deletions(-) diff --git a/local/rest_api_skeleton/Helpers/for_requests.py b/local/rest_api_skeleton/Helpers/for_requests.py index db53093d..9851264f 100644 --- a/local/rest_api_skeleton/Helpers/for_requests.py +++ b/local/rest_api_skeleton/Helpers/for_requests.py @@ -1,26 +1,47 @@ from flask_restful import reqparse -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage + def title_query(): query = reqparse.RequestParser() - query.add_argument("title", required=True, location="form" ) - return query.parse_args() + query.add_argument("title", required=True, location="form") + return query.parse_args() + def classifier_query(): query = reqparse.RequestParser() - query.add_argument("classifiers",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "classifiers", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def disturbance_query(): query = reqparse.RequestParser() - query.add_argument("disturbances",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() + query.add_argument( + "disturbances", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() + def miscellaneous_query(): query = reqparse.RequestParser() - query.add_argument("miscellaneous",type=FileStorage, required=True, action="append", location="files" ) - query.add_argument("attributes", location= "form") - return query.parse_args() - + query.add_argument( + "miscellaneous", + type=FileStorage, + required=True, + action="append", + location="files", + ) + query.add_argument("attributes", location="form") + return query.parse_args() diff --git a/local/rest_api_skeleton/Helpers/preprocess.py b/local/rest_api_skeleton/Helpers/preprocess.py index 157a0081..a496b64a 100644 --- a/local/rest_api_skeleton/Helpers/preprocess.py +++ b/local/rest_api_skeleton/Helpers/preprocess.py @@ -3,7 +3,8 @@ import json import rasterio as rst -def launch_run(title, input_dir): + +def launch_run(title, input_dir): s = time.time() with open(f"{input_dir}/gcbm_logs.csv", "w+") as f: res = subprocess.Popen( @@ -36,32 +37,43 @@ def launch_run(title, input_dir): "response": "Operation executed successfully. Downloadable links for input and output are attached in the response. Alternatively, you may also download this simulation input and output results by making a request at gcbm/download with the title in the body.", } -class Configs: +class Configs: def __init__(self, input_dir) -> None: self.input_dir = input_dir self.Rasters = [] - self.Rastersm = [] + self.Rastersm = [] self.nodatam = [] self.nodata = [] self.cellLatSize = [] self.cellLonSize = [] self.paths = [] self.lst = [] - self.provider_config = open(f"{os.getcwd()}/templates/provider_config.json", "r+") + self.provider_config = open( + f"{os.getcwd()}/templates/provider_config.json", "r+" + ) def get_config_templates(self): if not os.path.exists(f"{self.input_dir}/templates"): shutil.copytree( - f"{os.getcwd()}/templates", f"{self.input_dir}/templates", dirs_exist_ok=False + f"{os.getcwd()}/templates", + f"{self.input_dir}/templates", + dirs_exist_ok=False, + ) + self.provider_config = open( + f"{self.input_dir}/templates/provider_config.json", "r+" ) - self.provider_config = open(f"{self.input_dir}/templates/provider_config.json", "r+") def get_modules_cbm_config(self): - with open(f"{self.input_dir}/templates/modules_cbm.json", "r+") as modules_cbm_config: + with open( + f"{self.input_dir}/templates/modules_cbm.json", "r+" + ) as modules_cbm_config: data = json.load(modules_cbm_config) - disturbances = [file.split(".")[0][:-5] for file in os.listdir(f"{self.input_dir}/disturbances/")] # drop `_moja` to match modules_cbm.json template - modules_cbm_config.seek(0) + disturbances = [ + file.split(".")[0][:-5] + for file in os.listdir(f"{self.input_dir}/disturbances/") + ] # drop `_moja` to match modules_cbm.json template + modules_cbm_config.seek(0) data["Modules"]["CBMDisturbanceListener"]["settings"]["vars"] = disturbances json.dump(data, modules_cbm_config, indent=4) modules_cbm_config.truncate() @@ -70,11 +82,10 @@ def get_modules_cbm_config(self): def database_writes(self): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/db/"): - data["Providers"]["SQLite"] = {"type": "SQLite", "path": file } + data["Providers"]["SQLite"] = {"type": "SQLite", "path": file} self.provider_config.seek(0) - - def write_configs(self, config_type : str): + def write_configs(self, config_type: str): data = json.load(self.provider_config) for file in os.listdir(f"{self.input_dir}/{config_type}/"): d = dict() @@ -82,7 +93,7 @@ def write_configs(self, config_type : str): d["layer_path"] = file d["layer_prefix"] = file[:-5] self.lst.append(d) - if config_type == "disturbances" or config_type == "classifiers": + if config_type == "disturbances" or config_type == "classifiers": for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/{config_type}/") ): @@ -91,15 +102,15 @@ def write_configs(self, config_type : str): self.Rasters.append(fp) self.paths.append(fp) for self.nd in self.Rasters: - img = rst.open(self.nd) - t = img.transform + img = rst.open(self.nd) + t = img.transform x = t[0] y = -t[4] - n = img.nodata + n = img.nodata self.cellLatSize.append(x) self.cellLonSize.append(y) self.nodata.append(n) - result = all(element == self.cellLatSize[0] for element in self.cellLatSize) + result = all(element == self.cellLatSize[0] for element in self.cellLatSize) if result: cellLat = x cellLon = y @@ -112,8 +123,15 @@ def write_configs(self, config_type : str): print("Corrupt files") self.provider_config.seek(0) - new_values = {"cellLonSize": cellLon,"cellLatSize": cellLat, "blockLonSize": blockLon, "blockLatSize": blockLat, "tileLatSize": tileLat, "tileLonSize": tileLon} - data["Providers"]["RasterTiled"]["layers"] = self.lst + new_values = { + "cellLonSize": cellLon, + "cellLatSize": cellLat, + "blockLonSize": blockLon, + "blockLatSize": blockLat, + "tileLatSize": tileLat, + "tileLonSize": tileLon, + } + data["Providers"]["RasterTiled"]["layers"] = self.lst data["Providers"]["RasterTiled"].update(new_values) json.dump(data, self.provider_config, indent=4) @@ -132,21 +150,23 @@ def write_configs(self, config_type : str): } self.study_area = { - "tile_size": tileLat, - "block_size": blockLat, - "tiles": [ - { - "x": int(t[2]), - "y": int(t[5]), - "index": 12674, - } - ], - "pixel_size": cellLat, - "layers": [], - } + "tile_size": tileLat, + "block_size": blockLat, + "tiles": [ + { + "x": int(t[2]), + "y": int(t[5]), + "index": 12674, + } + ], + "pixel_size": cellLat, + "layers": [], + } def add_file_to_path(self, config_type): - for root, _, files in os.walk(os.path.abspath(f"{self.input_dir}/{config_type}/")): + for root, _, files in os.walk( + os.path.abspath(f"{self.input_dir}/{config_type}/") + ): for file in files: fp = os.path.join(root, file) self.paths.append(fp) @@ -157,15 +177,18 @@ def copy_directory(self): def flatten_directory(self, config_type): shutil.rmtree((f"{self.input_dir}/{config_type}/")) - + + class DisturbanceConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict = None ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict = None) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def disturbances_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) with open( @@ -194,14 +217,17 @@ def __call__(self): self.add_file_to_path("disturbances") self.copy_directory() + class ClassifierConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute def classifier_special(self): - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["attributes"] = self.attribute json.dump(self.dictionary, json_file, indent=4) @@ -225,16 +251,17 @@ def __call__(self): self.add_file_to_path("classifiers") self.copy_directory() + class MiscellaneousConfig(Configs): - def __init__(self, input_dir, config_file : str, attribute : dict ) -> None: + def __init__(self, input_dir, config_file: str, attribute: dict) -> None: super().__init__(input_dir) self.config_file = config_file self.attribute = attribute - - def miscellaneous_special(self): + + def miscellaneous_special(self): for root, _, files in os.walk( os.path.abspath(f"{self.input_dir}/miscellaneous/") -): + ): for file in files: fp2 = os.path.join(root, file) self.Rastersm.append(fp2) @@ -244,9 +271,11 @@ def miscellaneous_special(self): d = img.nodata self.nodatam.append(d) """this is an experimental thing""" - with open(f"{self.input_dir}/{self.config_file}", "w", encoding="utf8") as json_file: + with open( + f"{self.input_dir}/{self.config_file}", "w", encoding="utf8" + ) as json_file: self.dictionary["layer_type"] = "GridLayer" - if self.config_file == "mean_annual_temperature_moja.json": + if self.config_file == "mean_annual_temperature_moja.json": self.dictionary["layer_data"] = "Float32" else: self.dictionary["layer_data"] = "Int16" @@ -273,5 +302,3 @@ def __call__(self): self.add_file_to_path("miscellaneous") self.copy_directory() self.flatten_directory("miscellaneous") - - diff --git a/local/rest_api_skeleton/app.py b/local/rest_api_skeleton/app.py index c5502bc0..cd256cc9 100644 --- a/local/rest_api_skeleton/app.py +++ b/local/rest_api_skeleton/app.py @@ -2,8 +2,8 @@ from flask_restful import Api from Endpoints.gcbm_endpoints import disturbance, Run, title, classifier, miscellaneous -app = Flask(__name__) -api = Api() +app = Flask(__name__) +api = Api() api.add_resource(disturbance, "/gcbm/upload/disturbances") api.add_resource(title, "/gcbm/create") @@ -12,5 +12,5 @@ api.add_resource(Run, "/gcbm/run") -if __name__ == "__main__": +if __name__ == "__main__": app.run(debug=True)