diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b694934 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.venv \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9dcb600..0c2269f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ validated.yaml merged.yaml polytope_server.egg-info **/build -.venv \ No newline at end of file +.venv +skaffold.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6b2cc3e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,336 @@ +## +## Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +## In applying this licence, ECMWF does not waive the privileges and immunities +## granted to it by virtue of its status as an intergovernmental organisation nor +## does it submit to any jurisdiction. +## + +ARG fdb_base=blank-base +ARG mars_base_c=blank-base +ARG mars_base_cpp=blank-base +ARG gribjump_base=blank-base + +####################################################### +# C O M M O N +# based on alpine linux +####################################################### + +FROM python:3.11-alpine AS polytope-common + +RUN apk add --no-cache --virtual .build-deps gcc musl-dev openldap openldap-dev curl + +RUN set -eux \ + && addgroup --system polytope --gid 474 \ + && adduser --system polytope --ingroup polytope --home /home/polytope \ + && mkdir -p /polytope && chmod -R 777 /polytope \ + && mkdir -p /data && chmod -R 777 /data + +COPY ./requirements.txt /polytope/ +WORKDIR /polytope +RUN python -m pip install -r requirements.txt +COPY . /polytope/ +RUN python -m pip install --upgrade . + +####################################################### +# N O O P I M A G E +####################################################### +FROM python:3.11-bookworm AS blank-base +# create blank directories to copy from in the final stage, optional dependencies aren't built +RUN set -eux \ + && mkdir -p /root/.local \ + && mkdir -p /opt/ecmwf/mars-client \ + && mkdir -p /opt/ecmwf/mars-client-cpp \ + && mkdir -p /opt/ecmwf/mars-client-cloud \ + && mkdir -p /opt/fdb \ + && mkdir -p /opt/fdb-gribjump \ + && touch /usr/local/bin/mars + +####################################################### +# F D B B U I L D +####################################################### +FROM python:3.11-bookworm AS fdb-base +RUN apt update +# COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default + +# Install FDB from open source repositories +RUN set -eux && \ + apt install -y cmake gnupg build-essential libtinfo5 net-tools libnetcdf19 libnetcdf-dev bison flex && \ + rm -rf source && \ + rm -rf build && \ + mkdir -p source && \ + mkdir -p build && \ + mkdir -p /opt/fdb/ + +# Download ecbuild +RUN set -eux && \ + git clone --depth 1 --branch 3.8.2 https://github.com/ecmwf/ecbuild.git /ecbuild + +ENV PATH=/ecbuild/bin:$PATH + +# Install eckit +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/eckit.git /source/eckit && \ + cd /source/eckit && git checkout develop && \ + mkdir -p /build/eckit && \ + cd /build/eckit && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/eckit && \ + make -j4 && \ + make install + +# Install eccodes +RUN set -eux && \ + git clone --depth 1 --branch 2.33.1 https://github.com/ecmwf/eccodes.git /source/eccodes && \ + mkdir -p /build/eccodes && \ + cd /build/eccodes && \ + ecbuild --prefix=/opt/fdb -DENABLE_FORTRAN=OFF -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/eccodes && \ + make -j4 && \ + make install + +# Install metkit +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/metkit.git /source/metkit && \ + cd /source/metkit && git checkout develop && \ + mkdir -p /build/metkit && \ + cd /build/metkit && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH=/opt/fdb /source/metkit && \ + make -j4 && \ + make install + +# Install fdb \ +RUN set -eux && \ + git clone --depth 1 --branch develop https://github.com/ecmwf/fdb.git /source/fdb && \ + cd /source/fdb && git checkout develop && \ + mkdir -p /build/fdb && \ + cd /build/fdb && \ + ecbuild --prefix=/opt/fdb -- -DCMAKE_PREFIX_PATH="/opt/fdb;/opt/fdb/eckit;/opt/fdb/metkit" /source/fdb && \ + make -j4 && \ + make install + +RUN set -eux && \ + rm -rf /source && \ + rm -rf /build + +ARG ssh_prv_key +ARG ssh_pub_key + +# Install pyfdb \ +RUN set -eux \ + && git clone --single-branch --branch 0.0.3 https://github.com/ecmwf/pyfdb.git \ + && python -m pip install ./pyfdb --user + +####################################################### +# G R I B J U M P B U I L D +####################################################### + +FROM python:3.11-bookworm AS gribjump-base +ARG rpm_repo + +RUN response=$(curl -s -w "%{http_code}" ${rpm_repo}) \ + && if [ "$response" = "403" ]; then echo "Unauthorized access to ${rpm_repo} "; fi + +RUN set -eux \ + && apt-get update \ + && apt-get install -y gnupg2 curl ca-certificates \ + && curl -fsSL "${rpm_repo}/private-raw-repos-config/debian/bookworm/stable/public.gpg.key" | gpg --dearmor -o /usr/share/keyrings/mars-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/mars-archive-keyring.gpg] ${rpm_repo}/private-debian-bookworm-stable/ bookworm main" | tee /etc/apt/sources.list.d/mars.list + +RUN set -eux \ + && apt-get update \ + && apt install -y gribjump-client=0.5.3-gribjump + +RUN set -eux \ + ls -R /opt + +# gribjump not open source yet, clone it. +RUN set -eux \ + && git clone --single-branch --branch develop https://github.com/ecmwf/gribjump.git +# Install pygribjump +RUN set -eux \ +&& cd /gribjump \ + && python -m pip install . --user \ + && rm -rf /gribjump + +####################################################### +# F D B R E M O T E B U I L D +####################################################### + +FROM fdb-base AS fdb-remote-base + +ARG ssh_prv_key +ARG ssh_pub_key + +# gribjump not open source yet, clone it. +# COPY fdbremote /source/fdbremote +# Install fdb from local source (need a private version from internal ECMWF repository) +RUN apt-get install -y gfortran +RUN set -eux \ + # Configure SSH for private repository access + && mkdir -p /root/.ssh \ + && echo "$ssh_prv_key" > /root/.ssh/id_rsa \ + && echo "$ssh_pub_key" > /root/.ssh/id_rsa.pub \ + && chmod 600 /root/.ssh/id_rsa \ + && chmod 600 /root/.ssh/id_rsa.pub \ + && echo "StrictHostKeyChecking=no" > /root/.ssh/config \ + && mkdir -p build \ + && mkdir -p source \ + && mkdir -p /build/fdb \ + && cd /build/fdb \ + && ecbuild --prefix=/opt/fdbremote -- /source/fdbremote \ + && make -j4 \ + && make install \ + && rm -rf /build \ + && rm -rf /source \ + && rm -rf /root/.ssh + +####################################################### +# M A R S B A S E +####################################################### +FROM python:3.11-bookworm AS mars-base +ARG rpm_repo +ARG mars_client_cpp_version + +RUN response=$(curl -s -w "%{http_code}" ${rpm_repo}) \ + && if [ "$response" = "403" ]; then echo "Unauthorized access to ${rpm_repo} "; fi + +RUN set -eux \ + && curl -o stable-public.gpg.key "${rpm_repo}/private-raw-repos-config/debian/bookworm/stable/public.gpg.key" \ + && echo "deb ${rpm_repo}/private-debian-bookworm-stable/ bookworm main" >> /etc/apt/sources.list \ + && apt-key add stable-public.gpg.key \ + && apt-get update \ + && apt install -y libnetcdf19 liblapack3 + +FROM mars-base AS mars-base-c +RUN apt update && apt install -y liblapack3 mars-client=6.33.20.2 mars-client-cloud + +FROM mars-base AS mars-base-cpp +RUN apt update && apt install -y mars-client-cpp=${mars_client_cpp_version} +RUN set -eux \ + && python3 -m pip install git+https://github.com/ecmwf/pyfdb.git@master --user + +FROM blank-base AS blank-base-c +FROM blank-base AS blank-base-cpp + +####################################################### +# S W I T C H B A S E I M A G E S +####################################################### + +FROM ${fdb_base} AS fdb-base-final + +FROM ${mars_base_c} AS mars-c-base-final + +FROM ${mars_base_cpp} AS mars-cpp-base-final + +FROM ${gribjump_base} AS gribjump-base-final + + +####################################################### +# P Y T H O N R E Q U I R E M E N T S +####################################################### +FROM python:3.11-slim-bookworm AS worker-base + +# contains compilers for building wheels which we don't want in the final image +RUN apt update +RUN apt-get install -y --no-install-recommends gcc libc6-dev make gnupg2 +COPY ./requirements.txt /requirements.txt +RUN python -m pip install -r requirements.txt --user +RUN python -m pip install geopandas --user + +####################################################### +# W O R K E R +# based on debian bookworm +####################################################### + +FROM python:3.11-slim-bookworm AS worker + +ARG mars_config_branch +ARG mars_config_repo +ARG ssh_prv_key +ARG ssh_pub_key +ARG rpm_repo + + +RUN set -eux \ + && addgroup --system polytope --gid 474 \ + && adduser --system polytope --ingroup polytope --home /home/polytope \ + && mkdir /polytope && chmod -R o+rw /polytope + +RUN apt update +RUN apt install -y curl nano sudo ssh libgomp1 vim + +# Add polytope user to passwordless sudo group during build +RUN usermod -aG sudo polytope +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +WORKDIR /polytope +USER polytope + + +# Copy MARS-related artifacts +COPY --chown=polytope ./aux/mars-wrapper.py /polytope/bin/mars-wrapper.py +COPY --chown=polytope ./aux/mars-wrapper-docker.py /polytope/bin/mars-wrapper-docker.py + +COPY --chown=polytope --from=mars-cpp-base-final /opt/ecmwf/mars-client-cpp /opt/ecmwf/mars-client-cpp +COPY --chown=polytope --from=mars-cpp-base-final /root/.local /home/polytope/.local +COPY --chown=polytope --from=mars-c-base-final /opt/ecmwf/mars-client /opt/ecmwf/mars-client +COPY --chown=polytope --from=mars-c-base-final /usr/local/bin/mars /usr/local/bin/mars +RUN sudo apt update +RUN sudo apt install -y libgomp1 git libnetcdf19 liblapack3 libfftw3-bin libproj25 + + +# all of this is needed by the C client, would be nice to remove it at some point +RUN set -eux \ + && mkdir -p /home/polytope/.ssh \ + && chmod 0700 /home/polytope/.ssh \ + && ssh-keyscan git.ecmwf.int > /home/polytope/.ssh/known_hosts \ + && echo "$ssh_prv_key" > /home/polytope/.ssh/id_rsa \ + && echo "$ssh_pub_key" > /home/polytope/.ssh/id_rsa.pub \ + && chmod 600 /home/polytope/.ssh/id_rsa \ + && chmod 600 /home/polytope/.ssh/id_rsa.pub \ + && chmod 755 /polytope/bin/mars-wrapper.py \ + && chmod 755 /polytope/bin/mars-wrapper-docker.py + +ENV MARS_CONFIGS_REPO=${mars_config_repo} +ENV MARS_CONFIGS_BRANCH=${mars_config_branch} +ENV PATH="/polytope/bin/:/opt/ecmwf/mars-client/bin:/opt/ecmwf/mars-client-cloud/bin:${PATH}" + +# Copy FDB-related artifacts +COPY --chown=polytope --from=fdb-base-final /opt/fdb/ /opt/fdb/ +COPY --chown=polytope ./aux/default_fdb_schema /polytope/config/fdb/default +RUN mkdir -p /polytope/fdb/ && sudo chmod -R o+rw /polytope/fdb +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/fdb/lib:/opt/ecmwf/gribjump-client/lib +COPY --chown=polytope --from=fdb-base-final /root/.local /home/polytope/.local + +# COPY --chown=polytope --from=fdb-remote-base /opt/fdbremote/ /opt/fdbremote/ + +# Copy gribjump-related artifacts, including python libraries +# COPY --chown=polytope --from=gribjump-base-final /opt/fdb/ /opt/fdb/ +COPY --chown=polytope --from=gribjump-base-final /opt/ecmwf/gribjump-client/ /opt/ecmwf/gribjump-client/ +COPY --chown=polytope --from=gribjump-base-final /root/.local /home/polytope/.local +# RUN sudo apt install -y libopenjp2-7 +# COPY polytope-deployment/common/default_fdb_schema /polytope/config/fdb/default + +# Copy python requirements +COPY --chown=polytope --from=worker-base /root/.local /home/polytope/.local + + +# Install the server source +COPY --chown=polytope . /polytope/ + +RUN set -eux \ + && mkdir /home/polytope/data + +# Remove itself from sudo group +RUN sudo deluser polytope sudo \ No newline at end of file diff --git a/aux/default_fdb_schema b/aux/default_fdb_schema new file mode 100644 index 0000000..625f87e --- /dev/null +++ b/aux/default_fdb_schema @@ -0,0 +1,590 @@ +# From https://github.com/ecmwf/fdb/blob/c529bf5293d1f5e33048b1b12e7fdfbf4d7448b2/tests/fdb/etc/fdb/schema + + +# * Format of the rules is: + +# [a1, a2, a3 ...[b1, b2, b3... [c1, c2, c3...]]] + +# - The first level (a) defines which attributes are used to name the top level directory +# - The second level (b) defines which attributes are used to name the data files +# - The third level (c) defines which attributes are used as index keys + +# * Rules can be grouped + +# [a1, a2, a3 ... +# [b1, b2, b3... [c1, c2, c3...]] +# [B1, B2, B3... [C1, C2, C3...]] +# ] + +# * A list of values can be given for an attribute +# [ ..., stream=enfo/efov, ... ] +# This will be used when matching rules. + +# * Attributes can be typed +# Globally, at the begining of this file: + +# refdate: Date; + +# or in the context of a rule: +# [type=cl, ... [date:ClimateMonth, ...]] + +# Typing attributes is done when the user's requests or the GRIB values need to be modified before directories, files and indexes are created. For example, ClimateMonth will transform 2010-04-01 to 'may' internally. + +# * Attributes can be optional +# [ step, levelist?, param ] +# They will be replaced internally by an empty value. It is also posiible to provide a default subtitution value: e.g. [domain?g] will consider the domain to be 'g' if missing. + +# * Attributes can be removed: +# [grid-] +# This is useful to remove attributes present in the GRIB that should not be ignored + +# * Rules are matched: + +# - If the attributes are present in the GRIB/Request, or marked optional or ignored +# - If a list of possible value is provided, one of them must match, for example +# [ class, expver, stream=enfo/efov, date, time, domain ] +# will match either stream=enfo or stream=efov, all other attributes will be matched if they exist in the GRIB or user's request + +# * On archive: +# - Attributes are extracted from the GRIB (namespace 'mars'), possibly modified by the attribute type +# - Only the first rule is used, so order is important +# - All GRIB attributes must be used by the rules, otherwise an error is raised + +# * On retrieve: +# - Attributes are extracted from the user's request, possibly modified by the attribute type (e.g. for handling of U/V) +# - All the matching rules are considered +# - Only attributes listed in the rules are used to extract values from the user's request + + +# Default types + +param: Param; +step: Step; +date: Date; +hdate: Date; +refdate: Date; +latitude: Double; +longitude: Double; +levelist: Double; +grid: Grid; +expver: Expver; + +time: Time; +fcmonth: Integer; + +number: Integer; +frequency: Integer; +direction: Integer; +channel: Integer; + +instrument: Integer; +ident: Integer; + +diagnostic: Integer; +iteration: Integer; +system: Integer; +method: Integer; + +# ??????? + +# reference: Integer; +# fcperiod: Integer; + +# opttime: Integer; +# leadtime: Integer; + +# quantile: ?????? +# range: ?????? + +# band: Integer; + + +######################################################## +# These rules must be first, otherwise fields of These +# classes will be index with the default rule for oper +[ class=ti/s2, expver, stream, date, time, model + [ origin, type, levtype, hdate? + [ step, number?, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=de + [ domain, type, levtype, dbase, rki, rty, ty + [ step, levelist?, param ]] +] + +[ class=ms, expver, stream, date, time, country=it + [ domain, type, levtype, model, bcmodel, icmodel:First3 + [ step, levelist?, param ] + ] +] + +[ class=el, expver, stream, date, time, domain + [ origin, type, levtype + [ step, levelist?, param ]] +] + +######################################################## +# The are the rules matching most of the fields +# oper/dcda +[ class, expver, stream=oper/dcda/scda, date, time, domain? + + [ type=im/sim + [ step?, ident, instrument, channel ]] + + [ type=ssd + [ step, param, ident, instrument, channel ]] + + [ type=4i, levtype + [ step, iteration, levelist, param ]] + + [ type=me, levtype + [ step, number, levelist?, param ]] + + [ type=ef, levtype + [ step, levelist?, param, channel? ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype + [ step, levelist?, param ]] + +] + +# dcwv/scwv/wave +[ class, expver, stream=dcwv/scwv/wave, date, time, domain + [ type, levtype + [ step, param, frequency?, direction? ]]] + +# enfo +[ class, expver, stream=enfo/efov, date, time, domain + + [ type, levtype=dp, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type=tu, levtype, reference + [ step, number, levelist?, param ]] + + [ type, levtype + [ step, quantile?, number?, levelist?, param ]] + +] + +# waef/weov +[ class, expver, stream=waef/weov, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# enda +[ class, expver, stream=enda, date, time, domain + + [ type=ef/em/es/ses, levtype + [ step, number?, levelist?, param, channel? ]] + + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + + + [ type, levtype + [ step, number?, levelist?, param ]] +] + +# ewda +[ class, expver, stream=ewda, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] + + +######################################################## +# elda +[ class, expver, stream=elda, date, time, domain? + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, number?, levelist?, iteration?, param, channel? ]] +] + +# ewda +[ class, expver, stream=ewla, date, time, domain + [ type, levtype, anoffset + [ step, number?, param, frequency?, direction? ]] +] + +######################################################## +# elda +[ class, expver, stream=lwda, date, time, domain? + + [ type=ssd, anoffset + [ step, param, ident, instrument, channel ]] + + [type=me, levtype, anoffset + [ number, step, levelist?, param]] + + [ type=4i, levtype, anoffset + [ step, iteration, levelist, param ]] + + [ type=ofb/mfb + [ obsgroup, reportype ]] + + [ type, levtype, anoffset + [ step, levelist?, param]] +] + +# ewda +[ class, expver, stream=lwwv, date, time, domain + [ type, levtype, anoffset + [ step, param, frequency?, direction? ]] +] +######################################################## +# amap +[ class, expver, stream=amap, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# maed +[ class, expver, stream=maed, date, time, domain + [ type, levtype, origin + [ step, levelist?, param ]]] + +# mawv +[ class, expver, stream=mawv, date, time, domain + [ type, levtype, origin + [ step, param, frequency?, direction? ]]] + +# cher +[ class, expver, stream=cher, date, time, domain + [ type, levtype + [ step, levelist, param ]]] + + +# efhc +[ class, expver, stream=efhc, refdate, time, domain + [ type, levtype, date + [ step, number?, levelist?, param ]]] + +# efho +[ class, expver, stream=efho, date, time, domain + [ type, levtype, hdate + [ step, number?, levelist?, param ]]] + + +# efhs +[ class, expver, stream=efhs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# wehs +[ class, expver, stream=wehs, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]]] + +# kwbc +[ class, expver, stream=kwbc, date, time, domain + [ type, levtype + [ step, number?, levelist?, param ]]] + +# ehmm +[ class, expver, stream=ehmm, date, time, domain + [ type, levtype, hdate + [ fcmonth, levelist?, param ]]] + + +# ammc/cwao/edzw/egrr/lfpw/rjtd/toga +[ class, expver, stream=ammc/cwao/edzw/egrr/lfpw/rjtd/toga/fgge, date, time, domain + [ type, levtype + [ step, levelist?, param ]]] + +######################################################################## + +# enfh +[ class, expver, stream=enfh, date, time, domain + + [ type, levtype=dp, hdate, product?, section? + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, hdate + [ step, number?, levelist?, param ]] +] + +# enwh +[ class, expver, stream=enwh, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]] +] + +######################################################################## +# sens +[ class, expver, stream=sens, date, time, domain + [ type, levtype + [ step, diagnostic, iteration, levelist?, param ]]] + +######################################################################## +# esmm +[ class, expver, stream=esmm, date, time, domain + [ type, levtype + [ fcmonth, levelist?, param ]]] +# ewhc +[ class, expver, stream=ewhc, refdate, time, domain + [ type, levtype, date + [ step, number?, param, frequency?, direction? ]]] + +######################################################################## +# ewho +[ class, expver, stream=ewho, date, time, domain + [ type, levtype, hdate + [ step, number?, param, frequency?, direction? ]]] + +# mfam +[ class, expver, stream=mfam, date, time, domain + + [ type=pb/pd, levtype, origin, system?, method + [ fcperiod, quantile, levelist?, param ]] + + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]] + +] + +# mfhm +[ class, expver, stream=mfhm, refdate, time, domain + [ type, levtype, origin, system?, method, date? + [ fcperiod, number?, levelist?, param ]]] +# mfhw +[ class, expver, stream=mfhw, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ step, number?, param ]]] +# mfwm +[ class, expver, stream=mfwm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number, param ]]] +# mhwm +[ class, expver, stream=mhwm, refdate, time, domain + [ type, levtype, origin, system?, method, date + [ fcperiod, number, param ]]] + +# mmsf +[ class, expver, stream=mmsf, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +# mnfc +[ class, expver, stream=mnfc, date, time, domain + + [ type, levtype=dp, origin, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, origin, system?, method + [ step, number?, levelist?, param ]] +] + +# mnfh +[ class, expver, stream=mnfh, refdate, time, domain + [ type, levtype=dp, origin, product, section, system?, method, date + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, origin, system?, method, date? + [ step, number?, levelist?, param ]] +] + +# mnfm +[ class, expver, stream=mnfm, date, time, domain + [ type, levtype, origin, system?, method + [ fcperiod, number?, levelist?, param ]]] + +# mnfw +[ class, expver, stream=mnfw, date, time, domain + [ type, levtype, origin, system?, method + [ step, number?, param ]]] + +# ea/mnth +[ class=ea, expver, stream=mnth, date, domain + [ type, levtype + [ time, step?, levelist?, param ]]] + +# mnth +[ class, expver, stream=mnth, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, levelist?, param ]] + [ type, levtype + [ date , time, step?, levelist?, param ]]] + +# mofc +[ class, expver, stream=mofc, date, time, domain + [ type, levtype=dp, product, section, system?, method + [ step, number?, levelist?, latitude?, longitude?, range?, param ]] + [ type, levtype, system?, method + [ step, number?, levelist?, param ]] +] + +# mofm +[ class, expver, stream=mofm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, levelist?, param ]]] + +# mmsa/msmm +[ class, expver, stream=mmsa, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +[ class, expver, stream=msmm, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# ocea +[ class, expver, stream=ocea, date, time, domain + [ type, levtype, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] +] + +#=# seas +[ class, expver, stream=seas, date, time, domain + + [ type, levtype=dp, product, section, system?, method + [ step, number, levelist?, latitude?, longitude?, range?, param ]] + + [ type, levtype, system?, method + [ step, number, levelist?, param ]] +] + +# sfmm/smma +[ class, expver, stream=sfmm/smma, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number?, levelist?, param ]]] + +# supd +[ class=od, expver, stream=supd, date, time, domain + [ type, levtype, origin?, grid + [ step, levelist?, param ]]] + +# For era +[ class, expver, stream=supd, date, time, domain + [ type, levtype, grid- # The minus sign is here to consume 'grid', but don't index it + [ step, levelist?, param ]]] + +# swmm +[ class, expver, stream=swmm, date, time, domain + [ type, levtype, system?, method + [ fcmonth, number, param ]]] + +# wamf +[ class, expver, stream=wamf, date, time, domain + [ type, levtype, system?, method + [ step, number?, param ]]] + +# ea/wamo +[ class=ea, expver, stream=wamo, date, domain + [ type, levtype + [ time, step?, param ]]] + +# wamo +[ class, expver, stream=wamo, domain + [ type=cl, levtype + [ date: ClimateMonthly, time, param ]] + [ type, levtype + [ date, time, step?, param ]]] + +# wamd +[ class, expver, stream=wamd, date, domain + [ type, levtype + [ param ]]] + +# wasf +[ class, expver, stream=wasf, date, time, domain + [ type, levtype, system?, method + [ step, number, param ]]] +# wmfm +[ class, expver, stream=wmfm, date, time, domain + [ type, levtype, system?, method + [ fcperiod, number, param ]]] + +# moda +[ class, expver, stream=moda, date, domain + [ type, levtype + [ levelist?, param ]]] + +# msdc/mdfa/msda +[ class, expver, stream=msdc/mdfa/msda, domain + [ type, levtype + [ date, time?, step?, levelist?, param ]]] + + + +# seap +[ class, expver, stream=seap, date, time, domain + [ type=sv/svar, levtype, origin, method? + [ step, leadtime, opttime, number, levelist?, param ]] + + [ type=ef, levtype, origin + [ step, levelist?, param, channel? ]] + + [ type, levtype, origin + [ step, levelist?, param ]] + + ] + +[ class, expver, stream=mmaf, date, time, domain + [ type, levtype, origin, system?, method + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=mmam, date, time, domain + [ type, levtype, origin, system?, method + [ fcmonth, number, levelist?, param ]] +] + + +[ class, expver, stream=dacl, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, levelist?, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, levelist?, param ]] + +] + +[ class, expver, stream=dacw, domain + [ type=pb, levtype + [ date: ClimateDaily, time, step, quantile, param ]] + [ type, levtype + [ date: ClimateDaily, time, step, param ]] + +] + +[ class, expver, stream=edmm/ewmm, date, time, domain + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + [ type, levtype + [ step, number, levelist?, param ]] +] + +[ class, expver, stream=edmo/ewmo, date, domain + [ type, levtype + [ number, levelist?, param ]] +] + +# stream gfas +[ class=mc/rd, expver, stream=gfas, date, time, domain + [ type=ga, levtype + [ step, param ]] + + [ type=gsd + [ param, ident, instrument ]] + +] + +# class is e2 +[ class, expver, stream=espd, date, time, domain + [ type, levtype, origin, grid + [ step, number, levelist?, param ]]] + +[ class=cs, expver, stream, date:Default, time, domain + [ type, levtype + [ step, levelist?, param ]]] diff --git a/aux/mars-wrapper-docker.py b/aux/mars-wrapper-docker.py new file mode 100644 index 0000000..ec202d1 --- /dev/null +++ b/aux/mars-wrapper-docker.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + + +import logging +import os +import subprocess +import sys + +import docker + + +def main(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + assert len(sys.argv) == 2 + + c = docker.from_env() + # c = docker.DockerClient(base_url = ('tcp://' + + # os.environ['POLYTOPE_DOCKER_URL'])) + + container_name = os.environ["HOSTNAME"] + containers = c.containers.list() + cids = list(map(lambda x: x.short_id, containers)) + container = None + for cpos in range(len(cids)): + if container_name.startswith(cids[cpos]): + container = containers.pop(cpos) + break + if not container: + raise Exception("Container not found") + + container_port = str(os.environ.get("POLYTOPE_WORKER_MARS_LOCALPORT")) + external_port = container.ports[container_port + "/tcp"][0]["HostPort"] + swarm_node = container.labels["com.docker.swarm.node.id"] + nodes = c.nodes.list() + nids = list(map(lambda x: x.attrs["ID"], nodes)) + node = None + for npos in range(len(nids)): + if swarm_node == nids[npos]: + node = nodes.pop(npos) + break + if not node: + raise Exception("Node not found") + + node_name = node.attrs["Description"]["Hostname"] + + mars_command = os.environ.get("ECMWF_MARS_COMMAND", "mars") + + # Set the MARS client environment variables + + env = { + **os.environ, + "MARS_ENVIRON_ORIGIN": "polytope", + "MARS_DHS_CALLBACK_HOST": node_name, + "MARS_DHS_CALLBACK_PORT": external_port, + "MARS_DHS_LOCALPORT": container_port, + "MARS_DHS_LOCALHOST": node_name, + } + + # env = os.environ.copy() + + # def demote(user_uid, user_gid): + # def result(): + # report_ids('starting demotion') + # os.setgid(user_gid) + # os.setuid(user_uid) + # report_ids('finished demotion') + # return result + + # def report_ids(msg): + # print('uid, gid = %d, %d; %s' % (os.getuid(), os.getgid(), msg)) + + p = subprocess.Popen([mars_command, sys.argv[1]], cwd=os.path.dirname(__file__), shell=False, env=env) + return p.wait() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/aux/mars-wrapper.py b/aux/mars-wrapper.py new file mode 100644 index 0000000..781a841 --- /dev/null +++ b/aux/mars-wrapper.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + + +import logging +import os +import subprocess +import sys + +import requests + +port_file = "/persistent/last_mars_port" + + +def main(): + logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(name)s %(levelname)s %(message)s") + + assert len(sys.argv) == 2 + + # Read Kubernetes service account details for authentication + with open("/var/run/secrets/kubernetes.io/serviceaccount/token", "r") as file: + token = file.read().strip() + headers = {"Authorization": "Bearer " + token} + + # Set the MARS client environment variables + node_name = os.environ["K8S_NODE_NAME"] + pod_name = os.environ["K8S_POD_NAME"] # = service name + namespace = os.environ["K8S_NAMESPACE"] + + service_url = ( + f"https://{os.environ['KUBERNETES_SERVICE_HOST']}:" + f"{os.environ['KUBERNETES_PORT_443_TCP_PORT']}/api/v1/namespaces/" + f"{namespace}/services/{pod_name}" + ) + response = requests.get( + service_url, + headers=headers, + verify="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + ) + response.raise_for_status() # Raise an exception for HTTP errors + service = response.json()["spec"] + + try: + with open(port_file, "rt") as f: + last_port_id = int(f.read()) + except FileNotFoundError: + last_port_id = 0 + + port_id = (last_port_id + 1) % 5 + + with open(port_file, "w+") as f: + f.write(str(port_id)) + + node_port = service["ports"][port_id]["nodePort"] + local_port = service["ports"][port_id]["port"] + + logging.info("Callback on {}:{}".format(node_name, node_port)) + + env = { + **os.environ, + "MARS_ENVIRON_ORIGIN": "polytope", + "MARS_DHS_CALLBACK_HOST": node_name, + "MARS_DHS_CALLBACK_PORT": str(node_port), + "MARS_DHS_LOCALPORT": str(local_port), + "MARS_DHS_LOCALHOST": pod_name, + # "MARS_DEBUG": str(1), + # "ECKIT_DEBUG": str(1), + # "FDB_DEBUG": str(1), + } + + # Call MARS + mars_command = os.environ.get("ECMWF_MARS_COMMAND", "mars") + p = subprocess.Popen([mars_command, sys.argv[1]], cwd=os.path.dirname(__file__), shell=False, env=env) + return p.wait() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/polytope_server/common/datasource/polytope.py b/polytope_server/common/datasource/polytope.py index ac43dff..6718ab8 100644 --- a/polytope_server/common/datasource/polytope.py +++ b/polytope_server/common/datasource/polytope.py @@ -20,21 +20,11 @@ import json import logging -import os -import subprocess -from conflator import Conflator from polytope_mars.api import PolytopeMars -from polytope_mars.config import PolytopeMarsConfig -# os.environ["GRIBJUMP_HOME"] = "/opt/fdb-gribjump" -import tempfile -from pathlib import Path - -import polytope import yaml -from ..caching import cache from . import datasource @@ -47,32 +37,8 @@ def __init__(self, config): self.patch_rules = config.get("patch", {}) self.output = None - # still need to set up fdb - # self.fdb_config = self.config["fdb-config"] - - # self.non_sliceable = self.config.get("non-sliceable", None) - # assert self.non_sliceable is not None - self.polytope_options = self.config.get("polytope-options", {}) - - # self.check_schema() - - # # Set up gribjump - # self.gribjump_config = self.config["gribjump-config"] - # os.makedirs("/home/polytope/gribjump/", exist_ok=True) - # with open("/home/polytope/gribjump/config.yaml", "w") as f: - # json.dump(self.gribjump_config, f) - # os.environ["GRIBJUMP_CONFIG_FILE"] = "/home/polytope/gribjump/config.yaml" - # self.gj = pygribjump.GribJump() - - # Set up polytope feature extraction library - # self.polytope_options = { - # "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - # "step": {"type_change": "int"}, - # "number": {"type_change": "int"}, - # "longitude": {"cyclic": [0, 360]}, - # } + self.polytope_mars = PolytopeMars(self.config) logging.info("Set up gribjump") @@ -86,23 +52,7 @@ def retrieve(self, request): r = yaml.safe_load(request.user_request) logging.info(r) - # # We take the static config from the match rules of the datasource - # self.polytope_config = {} - # for k in self.non_sliceable: - # self.polytope_config[k] = r[k] - - # assert len(self.polytope_config) > 0 - - # logging.info(self.polytope_config) - # logging.info(self.polytope_options) - - conf = Conflator(app_name="polytope_mars", model=PolytopeMarsConfig).load() - cf = conf.model_dump() - cf["options"] = self.polytope_options - - p = PolytopeMars(cf, None) - - self.output = p.extract(r) + self.output = self.polytope_mars.extract(r) self.output = json.dumps(self.output).encode("utf-8") # logging.info(self.output) return True diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..9de22b5 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,45 @@ + +apiVersion: skaffold/v4beta10 +kind: Config + +# This Skaffold configuration expects to find a skaffold.env file in the current directory. +# It should contain the following environment variables: +# - SKAFFOLD_DEFAULT_REPO: The default repository to use for images +# - rpm_repo: The URL of the RPM repository for MARS/GribJump/FDB images +# - mars_config_repo: The URL of the MARS configuration repository +# - mars_config_branch: The branch of the MARS configuration repository to use +# - ssh_pub_key: The public SSH key to use for cloning the MARS and MARS configuration repositories +# - ssh_prv_key: The private SSH key to use for cloning the MARS and MARS configuration repositories + + +build: + local: + useBuildkit: True + concurrency: 1 + + tagPolicy: + gitCommit: {} + + artifacts: + + # Polytope common + - image: "polytope-common" + docker: + target: polytope-common + + # Worker with all clients (FDB, GribJump, MARS C, MARS CPP) + - image: "worker" + docker: + target: worker + buildArgs: + rpm_repo: "{{ .rpm_repo }}" + mars_config_repo: "{{ .mars_config_repo }}" + mars_config_branch: "{{ .mars_config_branch }}" + ssh_pub_key: "{{ .ssh_pub_key }}" + ssh_prv_key: "{{ .ssh_prv_key }}" + mars_client_cpp_version: 6.99.3.0 + mars_base_c: mars-base-c + mars_base_cpp: mars-base-cpp + fdb_base: blank-base #fdb-base + gribjump_base: gribjump-base +