From a79ea511bd7c5388c707e0b8b768b95ef33729cf Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 1 Mar 2023 20:30:51 +0100 Subject: [PATCH 01/59] chore: workflow clean-up --- .github/workflows/codeql.yml | 76 ------------------- .github/workflows/{docker.yml => release.yml} | 16 ++-- Makefile | 7 +- backend/api/order/order.py | 3 + backend/nebula/__init__.py | 8 ++ backend/pyproject.toml | 4 +- 6 files changed, 26 insertions(+), 88 deletions(-) delete mode 100644 .github/workflows/codeql.yml rename .github/workflows/{docker.yml => release.yml} (97%) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index a6e33ce..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,76 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - schedule: - - cron: '31 8 * * 0' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docker.yml b/.github/workflows/release.yml similarity index 97% rename from .github/workflows/docker.yml rename to .github/workflows/release.yml index 5cff3c8..c1c24a9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Publish docker image +name: Publish a new version on: push: @@ -10,6 +10,12 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Get the version id: get_version @@ -18,14 +24,6 @@ jobs: file: 'backend/pyproject.toml' field: 'tool.poetry.version' - - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build docker image uses: docker/build-push-action@v4 with: diff --git a/Makefile b/Makefile index 85eb7c1..051f2b4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ IMAGE_NAME=nebulabroadcast/nebula-server:latest +VERSION=$(shell cd backend && poetry run python -c 'import nebula' --version) -test: +check: check_version cd frontend && yarn format cd backend && \ @@ -9,6 +10,10 @@ test: poetry run flake8 . && \ poetry run mypy . +check_version: + echo $(VERSION) + sed -i "s/version = \".*\"/version = \"$(VERSION)\"/" backend/pyproject.toml + build: docker build -t $(IMAGE_NAME) . diff --git a/backend/api/order/order.py b/backend/api/order/order.py index cccd632..4a6981f 100644 --- a/backend/api/order/order.py +++ b/backend/api/order/order.py @@ -33,6 +33,9 @@ async def set_rundown_order( connection=conn, username=user.name, ) + + assert isinstance(item, nebula.Item) # mypy + # Empty event may not have id_bin set, # but we know, where we are putting it. item["id_bin"] = id_bin diff --git a/backend/nebula/__init__.py b/backend/nebula/__init__.py index ea595bd..f086aa0 100644 --- a/backend/nebula/__init__.py +++ b/backend/nebula/__init__.py @@ -1,3 +1,5 @@ +__version__ = "6.0.0-rc.1" + __all__ = [ "config", "settings", @@ -28,6 +30,12 @@ "CLIPlugin", ] +import sys + +if "--version" in sys.argv: + print(__version__) + sys.exit(0) + import asyncio from .config import config diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 32d69c0..05c0fa7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nebula" -version = "6.0.0-beta.2" +version = "6.0.0-rc.1" description = "Open source broadcast automation system" authors = ["Nebula Broadcast "] @@ -17,7 +17,7 @@ pydantic = "^1.10.4" python-dotenv = "^0.19.2" redis = "^4.5.1" rich = "^12.0.1" -uvicorn = {extras = ["standard"], version = "^0.17.6"} +uvicorn = {extras = ["standard"], version = "6.0.0-rc.1"} [tool.poetry.dev-dependencies] pytest = "^7.0" From 0f359216c86a20149706d2e986e095fbb56e8278 Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 1 Mar 2023 20:47:04 +0100 Subject: [PATCH 02/59] fix: transparent edges of table headers --- frontend/src/components/table/container.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/table/container.jsx b/frontend/src/components/table/container.jsx index 2dfc1d5..c7dbfea 100644 --- a/frontend/src/components/table/container.jsx +++ b/frontend/src/components/table/container.jsx @@ -10,7 +10,7 @@ const TableWrapper = styled.div` border-collapse: collapse; tr { - border-left: 2px solid black; + border-left: 2px solid var(--color-surface-02); } td, th { @@ -47,7 +47,7 @@ const TableWrapper = styled.div` position: sticky; top: 0; height: var(--input-height); - border-right: 2px solid var(--color-surface-02); + outline: 2px solid var(--color-surface-02); } :first-child { From 959015ef7c643633a5c95d230400305eb0315756 Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 1 Mar 2023 21:18:32 +0100 Subject: [PATCH 03/59] fix: asset editor changes --- frontend/src/pages/assetEditor/index.jsx | 6 ++---- frontend/src/pages/assetEditor/preview.jsx | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/pages/assetEditor/index.jsx b/frontend/src/pages/assetEditor/index.jsx index ffa8bd3..f408323 100644 --- a/frontend/src/pages/assetEditor/index.jsx +++ b/frontend/src/pages/assetEditor/index.jsx @@ -86,11 +86,9 @@ const AssetEditor = () => { }, [assetData, originalData]) const isChanged = useMemo(() => { - if (!originalData?.id) return false let changed = [] - for (const key in originalData) { - if (!isEqual(originalData[key], assetData[key])) { - console.log('CHANGE', key, originalData[key], assetData[key]) + for (const key in assetData) { + if (!isEqual(originalData[key] || null, assetData[key] || null)) { return true //changed.push(key) } diff --git a/frontend/src/pages/assetEditor/preview.jsx b/frontend/src/pages/assetEditor/preview.jsx index fbc6929..dae042f 100644 --- a/frontend/src/pages/assetEditor/preview.jsx +++ b/frontend/src/pages/assetEditor/preview.jsx @@ -107,7 +107,7 @@ const Preview = ({ assetData, setAssetData }) => { useEffect(() => { if (!assetData) return if ((assetData.subclips || []) !== subclips) { - setAssetData((assetData) => ({ ...assetData, subclips: subclips })) + setAssetData((assetData) => ({ ...assetData, subclips: subclips.length ? subclips : null })) } }, [subclips]) From 3100cfe6d2e730f03ff0714e752d02b35b731fa8 Mon Sep 17 00:00:00 2001 From: Martastain Date: Fri, 3 Mar 2023 23:21:36 +0100 Subject: [PATCH 04/59] updated setup defaults, fixes and clean-up --- Makefile | 2 +- backend/nebula/objects/user.py | 3 ++- backend/nebula/settings/models.py | 8 +++++++- backend/pyproject.toml | 2 +- backend/schema/meta-aliases-cs.json | 2 ++ backend/schema/meta-aliases-en.json | 2 ++ backend/server/__init__.py | 11 ++++++++++- backend/setup/defaults/meta_types.py | 12 +++++++++++- backend/setup/metatypes.py | 10 ++++++++-- backend/setup/settings.py | 14 +++++++++++++- frontend/src/pages/assetEditor/preview.jsx | 5 ++++- 11 files changed, 61 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 051f2b4..237222c 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ check: check_version check_version: echo $(VERSION) - sed -i "s/version = \".*\"/version = \"$(VERSION)\"/" backend/pyproject.toml + sed -i "s/^version = \".*\"/version = \"$(VERSION)\"/" backend/pyproject.toml build: docker build -t $(IMAGE_NAME) . diff --git a/backend/nebula/objects/user.py b/backend/nebula/objects/user.py index 19074d8..d39255b 100644 --- a/backend/nebula/objects/user.py +++ b/backend/nebula/objects/user.py @@ -4,6 +4,7 @@ import asyncpg from pydantic import BaseModel, Field +from nebula.settings import settings from nebula.config import config from nebula.db import db from nebula.exceptions import ( @@ -50,7 +51,7 @@ class User(BaseObject): @property def language(self): """Return the preferred language of the user.""" - return self["language"] or "en" + return self["language"] or settings.system.language @property def name(self): diff --git a/backend/nebula/settings/models.py b/backend/nebula/settings/models.py index b1ffc23..ea47c3c 100644 --- a/backend/nebula/settings/models.py +++ b/backend/nebula/settings/models.py @@ -3,7 +3,7 @@ from pydantic import Field from nebula.enum import ContentType, MediaType, ServiceState -from nebula.settings.common import SettingsModel +from nebula.settings.common import SettingsModel, LanguageCode from nebula.settings.metatypes import MetaType CSItemRole = Literal["hidden", "header", "label", "option"] @@ -59,6 +59,12 @@ class BaseSystemSettings(SettingsModel): description="A name used as the site (instance) identification", ) + language: LanguageCode = Field( + "en", + title="Default language", + example="en", + ) + ui_asset_create: bool = Field( True, title="Create assets in UI", diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 05c0fa7..90cdf59 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -17,7 +17,7 @@ pydantic = "^1.10.4" python-dotenv = "^0.19.2" redis = "^4.5.1" rich = "^12.0.1" -uvicorn = {extras = ["standard"], version = "6.0.0-rc.1"} +uvicorn = {extras = ["standard"], version = "0.20.0"} [tool.poetry.dev-dependencies] pytest = "^7.0" diff --git a/backend/schema/meta-aliases-cs.json b/backend/schema/meta-aliases-cs.json index fede682..b307cbb 100644 --- a/backend/schema/meta-aliases-cs.json +++ b/backend/schema/meta-aliases-cs.json @@ -74,6 +74,7 @@ ["graphic_usage" , "Užití grafiky" , "Užití grafiky" , ""], ["audio/silence" , "Ticho" , null , ""], ["place" , "Místo" , null , ""], + ["place/type" , "Typ místa" , null , ""], ["video/is_interlaced" , "Prokládání" , null , ""], ["id_item" , "Item ID" , null , ""], ["rights/spatial" , "Regionální omezení" , null , ""], @@ -97,6 +98,7 @@ ["id_folder" , "Složka" , null , ""], ["is_admin" , "Admin" , null , ""], ["run_mode" , "Run mode" , null , "Režim odbavení příspěvků"], + ["summary" , "Perex" , null , ""], ["description" , "Popis" , null , ""], ["file/format" , "Formát souboru" , null , ""], ["id" , "ID" , "#" , ""], diff --git a/backend/schema/meta-aliases-en.json b/backend/schema/meta-aliases-en.json index 1afaffb..10fcbbb 100644 --- a/backend/schema/meta-aliases-en.json +++ b/backend/schema/meta-aliases-en.json @@ -74,6 +74,7 @@ ["graphic_usage" , "Graphic usage" , "Usage" , ""], ["audio/silence" , "Silence" , null , ""], ["place" , "Place" , null , ""], + ["place/type" , "Place type" , null , ""], ["video/is_interlaced" , "Is interlaced" , "Interlaced" , ""], ["id_item" , "Item ID" , null , ""], ["rights/spatial" , "Spatial rights" , null , ""], @@ -97,6 +98,7 @@ ["id_folder" , "Folder" , null , ""], ["is_admin" , "Admin" , null , ""], ["run_mode" , "Run mode" , null , ""], + ["summary" , "Summary" , null , ""], ["description" , "Description" , null , ""], ["file/format" , "File format" , null , ""], ["id" , "ID" , "#" , ""], diff --git a/backend/server/__init__.py b/backend/server/__init__.py index 0c569d2..977ec99 100644 --- a/backend/server/__init__.py +++ b/backend/server/__init__.py @@ -1,4 +1,5 @@ import os +import time import aiofiles from fastapi import Depends, FastAPI, Header, Request @@ -166,16 +167,24 @@ async def upload_media_file( assert extension in ["mp4", "mov", "mxf"], "Invalid extension" nebula.log.debug(f"Uploading media file for {asset}", user=user.name) + + temp_dir = os.path.join(storage.local_path, ".nx", "creating") + if not os.path.isdir(temp_dir): + os.makedirs(temp_dir) + + temp_path = os.path.join(temp_dir, f"upload-{asset.id}-{time.time()}") + target_path = os.path.join( storage.local_path, upload_dir, f"{asset.id}.{extension}" ) i = 0 - async with aiofiles.open(target_path, "wb") as f: + async with aiofiles.open(temp_path, "wb") as f: async for chunk in request.stream(): i += len(chunk) await f.write(chunk) + os.rename(temp_path, target_path) nebula.log.info(f"Uploaded media file for {asset}", user=user.name) diff --git a/backend/setup/defaults/meta_types.py b/backend/setup/defaults/meta_types.py index b2a5389..ff8302f 100644 --- a/backend/setup/defaults/meta_types.py +++ b/backend/setup/defaults/meta_types.py @@ -189,12 +189,18 @@ "fulltext": 8, "type": T.STRING, }, - "description": { + "summary": { "ns": "m", "fulltext": 8, "type": T.TEXT, "syntax": "md", }, + "description": { + "ns": "m", + "fulltext": 7, + "type": T.TEXT, + "syntax": "md", + }, "color": { "ns": "m", "type": T.COLOR, @@ -269,6 +275,10 @@ "order": "alias", }, "place": { + "ns": "m", + "type": T.STRING, + }, + "place/type": { "ns": "m", "type": T.LIST, "cs": "urn:tva:metadata-cs:PlaceTypeCS", diff --git a/backend/setup/metatypes.py b/backend/setup/metatypes.py index 426749c..69f2d20 100644 --- a/backend/setup/metatypes.py +++ b/backend/setup/metatypes.py @@ -16,7 +16,6 @@ async def setup_metatypes(meta_types, db): aliases[lang][key] = [alias, header, description] for key, data in meta_types.items(): - meta_type = {} meta_type["ns"] = data["ns"] meta_type["editable"] = True @@ -28,7 +27,14 @@ async def setup_metatypes(meta_types, db): meta_type[opt] = data[opt] for lang in languages: - meta_type["aliases"][lang] = aliases[lang][key] + try: + meta_type["aliases"][lang] = aliases[lang][key] + except KeyError: + meta_type["aliases"][lang] = [ + data.get("title", key.capitalize()), + None, + "", + ] await db.execute( """ diff --git a/backend/setup/settings.py b/backend/setup/settings.py index dccd5a7..61caa4a 100644 --- a/backend/setup/settings.py +++ b/backend/setup/settings.py @@ -31,6 +31,7 @@ "meta_types": META_TYPES, "storages": [], "settings": {}, + "cs": [], } @@ -47,10 +48,11 @@ def load_overrides(): for key in TEMPLATE: if not hasattr(mod, key.upper()): continue - log.info(f"Using settings overrides: {spath}") override = getattr(mod, key.upper()) + log.info(f"Found overrides for {key}") if type(override) == dict and type(TEMPLATE[key]) == dict: + log.info("Updating settings", override) TEMPLATE[key].update(override) elif type(override) == list and type(TEMPLATE[key]) == list: TEMPLATE[key] = override @@ -144,13 +146,23 @@ async def setup_settings(db): # Setup classifications + used_urns = set() + for meta_type, mset in TEMPLATE["meta_types"].items(): + if mset.get("cs"): + used_urns.add(mset["cs"]) + classifications = [] async with httpx.AsyncClient() as client: response = await client.get("https://cs.nbla.xyz/dump") classifications = response.json() + classifications.extend(TEMPLATE["cs"]) + for scheme in classifications: name = scheme["cs"] + if name not in used_urns: + log.trace(f"Skipping unused classification scheme: {name}") + continue await db.execute("DELETE FROM cs WHERE cs = $1", name) for value in scheme["data"]: settings = scheme["data"][value] diff --git a/frontend/src/pages/assetEditor/preview.jsx b/frontend/src/pages/assetEditor/preview.jsx index dae042f..8a33ef2 100644 --- a/frontend/src/pages/assetEditor/preview.jsx +++ b/frontend/src/pages/assetEditor/preview.jsx @@ -107,7 +107,10 @@ const Preview = ({ assetData, setAssetData }) => { useEffect(() => { if (!assetData) return if ((assetData.subclips || []) !== subclips) { - setAssetData((assetData) => ({ ...assetData, subclips: subclips.length ? subclips : null })) + setAssetData((assetData) => ({ + ...assetData, + subclips: subclips.length ? subclips : null, + })) } }, [subclips]) From 6b4e6840ce0b46807546704a9628cb65fcc849a9 Mon Sep 17 00:00:00 2001 From: Martastain Date: Sun, 5 Mar 2023 21:04:07 +0100 Subject: [PATCH 05/59] support for direct uploads --- backend/server/__init__.py | 33 ++++++++++++++++++++++----------- backend/setup/defaults/views.py | 2 +- backend/setup/settings.py | 1 - 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/backend/server/__init__.py b/backend/server/__init__.py index 977ec99..c4acbc5 100644 --- a/backend/server/__init__.py +++ b/backend/server/__init__.py @@ -8,7 +8,7 @@ from fastapi.websockets import WebSocket, WebSocketDisconnect import nebula -from nebula.enum import MediaType +from nebula.enum import MediaType, ObjectStatus from nebula.exceptions import NebulaException from nebula.settings import load_settings from server.dependencies import asset_in_path, current_user, current_user_query @@ -156,16 +156,22 @@ async def upload_media_file( """ assert asset["media_type"] == MediaType.FILE, "Only file assets can be uploaded" - assert nebula.settings.system.upload_storage, "Upload storage not configured" - assert nebula.settings.system.upload_dir, "Upload path not configured" - - storage = nebula.storages[nebula.settings.system.upload_storage] - upload_dir = nebula.settings.system.upload_dir - extension = request.headers.get("X-nebula-extension") assert extension, "Missing X-nebula-extension header" assert extension in ["mp4", "mov", "mxf"], "Invalid extension" + if nebula.settings.system_upload_storage and nebula.settings.system_upload_dir: + direct = False + storage = nebula.storages[nebula.settings.system.upload_storage] + upload_dir = nebula.settings.system.upload_dir + target_path = os.path.join( + storage.local_path, upload_dir, f"{asset.id}.{extension}" + ) + else: + direct = True + storage = nebula.storages[asset["id_storage"]] + target_path = os.path.join(storage.local_path, f"{asset.id}.{extension}") + nebula.log.debug(f"Uploading media file for {asset}", user=user.name) temp_dir = os.path.join(storage.local_path, ".nx", "creating") @@ -174,10 +180,6 @@ async def upload_media_file( temp_path = os.path.join(temp_dir, f"upload-{asset.id}-{time.time()}") - target_path = os.path.join( - storage.local_path, upload_dir, f"{asset.id}.{extension}" - ) - i = 0 async with aiofiles.open(temp_path, "wb") as f: async for chunk in request.stream(): @@ -185,6 +187,15 @@ async def upload_media_file( await f.write(chunk) os.rename(temp_path, target_path) + if direct: + if extension != os.path.splitext(asset["path"])[1][1:]: + nebula.log.warning( + f"Uploaded media file extension {extension} does not match " + f"asset extension {os.path.splitext(asset['path'])[1][1:]}" + ) + asset["path"] = os.path.splitext(asset["path"])[0] + "." + extension + asset["status"] = ObjectStatus.CREATING + await asset.save() nebula.log.info(f"Uploaded media file for {asset}", user=user.name) diff --git a/backend/setup/defaults/views.py b/backend/setup/defaults/views.py index 9280f20..61e227f 100644 --- a/backend/setup/defaults/views.py +++ b/backend/setup/defaults/views.py @@ -78,7 +78,7 @@ name="Series", separator=True, position=30, - folders=[11], + folders=[13], columns=[ "title", "genre", diff --git a/backend/setup/settings.py b/backend/setup/settings.py index 61caa4a..84aad74 100644 --- a/backend/setup/settings.py +++ b/backend/setup/settings.py @@ -52,7 +52,6 @@ def load_overrides(): log.info(f"Found overrides for {key}") if type(override) == dict and type(TEMPLATE[key]) == dict: - log.info("Updating settings", override) TEMPLATE[key].update(override) elif type(override) == list and type(TEMPLATE[key]) == list: TEMPLATE[key] = override From d96f5039725a98f2de7d2723868638a260d946bd Mon Sep 17 00:00:00 2001 From: Martastain Date: Sun, 5 Mar 2023 22:27:25 +0100 Subject: [PATCH 06/59] fix: upload dir --- backend/nebula/objects/user.py | 2 +- backend/nebula/settings/models.py | 2 +- backend/server/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/nebula/objects/user.py b/backend/nebula/objects/user.py index d39255b..aa24fd8 100644 --- a/backend/nebula/objects/user.py +++ b/backend/nebula/objects/user.py @@ -4,7 +4,6 @@ import asyncpg from pydantic import BaseModel, Field -from nebula.settings import settings from nebula.config import config from nebula.db import db from nebula.exceptions import ( @@ -14,6 +13,7 @@ NotImplementedException, ) from nebula.objects.base import BaseObject +from nebula.settings import settings def hash_password(password: str): diff --git a/backend/nebula/settings/models.py b/backend/nebula/settings/models.py index ea47c3c..745e2d1 100644 --- a/backend/nebula/settings/models.py +++ b/backend/nebula/settings/models.py @@ -3,7 +3,7 @@ from pydantic import Field from nebula.enum import ContentType, MediaType, ServiceState -from nebula.settings.common import SettingsModel, LanguageCode +from nebula.settings.common import LanguageCode, SettingsModel from nebula.settings.metatypes import MetaType CSItemRole = Literal["hidden", "header", "label", "option"] diff --git a/backend/server/__init__.py b/backend/server/__init__.py index c4acbc5..be500d4 100644 --- a/backend/server/__init__.py +++ b/backend/server/__init__.py @@ -160,7 +160,7 @@ async def upload_media_file( assert extension, "Missing X-nebula-extension header" assert extension in ["mp4", "mov", "mxf"], "Invalid extension" - if nebula.settings.system_upload_storage and nebula.settings.system_upload_dir: + if nebula.settings.system.upload_storage and nebula.settings.system.upload_dir: direct = False storage = nebula.storages[nebula.settings.system.upload_storage] upload_dir = nebula.settings.system.upload_dir From c4d79076cd1f77cd3b0fd6d5e3a4f43a050a42ec Mon Sep 17 00:00:00 2001 From: Martastain Date: Mon, 6 Mar 2023 22:18:28 +0100 Subject: [PATCH 07/59] feat: support for sending emails --- backend/nebula/helpers/email.py | 68 +++++++++++++++++++++++++++++++ backend/nebula/settings/models.py | 5 ++- 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 backend/nebula/helpers/email.py diff --git a/backend/nebula/helpers/email.py b/backend/nebula/helpers/email.py new file mode 100644 index 0000000..d157cfa --- /dev/null +++ b/backend/nebula/helpers/email.py @@ -0,0 +1,68 @@ +import smtplib +import nebula + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +try: + import mistune # noqa + + has_mistune = True +except ModuleNotFoundError: + has_mistune = False + + +def html2email(html) -> MIMEMultipart: + msg = MIMEMultipart("alternative") + text = "no plaitext version available" + part1 = MIMEText(text, "plain") + part2 = MIMEText(html, "html") + + msg.attach(part1) + msg.attach(part2) + + return msg + + +def markdown2email(text) -> MIMEMultipart | MIMEText: + if has_mistune: + msg = MIMEMultipart("alternative") + html = mistune.html(text) + part1 = MIMEText(text, "plain") + part2 = MIMEText(html, "html") + msg.attach(part1) + msg.attach(part2) + return msg + else: + return MIMEText(text, "plain") + + +def send_mail(to: str, subject: str, body: str | MIMEText | MIMEMultipart, **kwargs): + if type(to) == str: + to = [to] + reply_address = kwargs.get("from", nebula.settings.system.mail_from) + + if isinstance(body, MIMEMultipart): + msg = body + else: + msg = MIMEText(body) + + msg["Subject"] = subject + msg["From"] = reply_address + msg["To"] = ",".join(to) + if nebula.settings.system.smtp_port == 25: + s = smtplib.SMTP(nebula.settings.system.smtp_host, port=25) + else: + s = smtplib.SMTP_SSL( + nebula.settings.system.smtp_host, port=nebula.settings.system.smtp_port + ) + + user = nebula.settings.system.smtp_user + password = nebula.settings.system.smtp_pass + + if user: + assert password is not None, "SMTP user set but no password" + + if user and password: + s.login(user, password) + s.sendmail(reply_address, [to], msg.as_string()) diff --git a/backend/nebula/settings/models.py b/backend/nebula/settings/models.py index 745e2d1..62db255 100644 --- a/backend/nebula/settings/models.py +++ b/backend/nebula/settings/models.py @@ -103,9 +103,10 @@ class SystemSettings(BaseSystemSettings): smtp_host: str | None = Field(None, title="SMTP host", example="smtp.example.com") smtp_port: int | None = Field(None, title="SMTP port", example=465) smtp_user: str | None = Field(None, title="SMTP user", example="smtpuser") - smtp_password: str | None = Field(None, title="SMTP password", example="smtppass.1") + smtp_pass: str | None = Field(None, title="SMTP password", example="smtppass.1") + mail_from: str | None = Field( - None, + "Nebula ", title="Mail from", description="Email address used as the sender", example="Nebula ", From 501f5b27685b8143836942643ed7198b25e34f53 Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 7 Mar 2023 00:01:23 +0100 Subject: [PATCH 08/59] password changing, sending emails --- backend/api/auth.py | 47 +++++++++++++++++++++++++++++++-- backend/cli/__main__.py | 4 ++- backend/mypy.ini | 5 ++++ backend/nebula/config.py | 5 +++- backend/nebula/helpers/email.py | 36 ++++++++++++++++++------- backend/server/__init__.py | 1 + 6 files changed, 85 insertions(+), 13 deletions(-) diff --git a/backend/api/auth.py b/backend/api/auth.py index 28a2be2..925d006 100644 --- a/backend/api/auth.py +++ b/backend/api/auth.py @@ -1,8 +1,9 @@ -from fastapi import Header +from fastapi import Depends, Header from pydantic import Field import nebula -from nebula.exceptions import UnauthorizedException +from nebula.exceptions import NotFoundException, UnauthorizedException +from server.dependencies import current_user from server.models import RequestModel, ResponseModel from server.request import APIRequest from server.session import Session @@ -37,6 +38,11 @@ class LoginResponseModel(ResponseModel): ) +class PasswordRequestModel(RequestModel): + login: str | None = Field(None, title="Login", example="admin") + password: str = Field(..., title="Password", example="Password.123") + + # # Request # @@ -71,3 +77,40 @@ async def handle(self, authorization: str | None = Header(None)): await Session.delete(access_token) raise UnauthorizedException("Logged out") + + +class SetPassword(APIRequest): + """Set a new password for the current (or a given) user. + + In order to set a password for another user, the current user must be an admin. + """ + + name: str = "password" + title: str = "Set password" + + async def handle( + self, + request: PasswordRequestModel, + user: nebula.User = Depends(current_user), + ): + if request.login: + if not user.is_admin: + raise UnauthorizedException( + "Only admin can change other user's password" + ) + query = "SELECT meta FROM users WHERE login = $1" + async for row in nebula.db.iterate(query, request.login): + target_user = nebula.User.from_row(row) + break + else: + raise NotFoundException(f"User {request.login} not found") + else: + target_user = user + + if len(request.password) < 8: + raise UnauthorizedException("Password is too short") + + target_user.set_password(request.password) + await user.save() + + return ResponseModel() diff --git a/backend/cli/__main__.py b/backend/cli/__main__.py index 7516be5..8aad255 100644 --- a/backend/cli/__main__.py +++ b/backend/cli/__main__.py @@ -24,9 +24,11 @@ def get_plugin(name: str): try: plugin_module = import_module(module_name, module_path) except ModuleNotFoundError: - nebula.log.error(f"Module {name} not found") + nebula.log.error(f"Unable to import module {module_path}") + continue except ImportError: nebula.log.traceback(f"Error importing module {name}") + continue for plugin_class in classes_from_module( nebula.plugins.CLIPlugin, plugin_module diff --git a/backend/mypy.ini b/backend/mypy.ini index 32ad10c..e34a1a3 100644 --- a/backend/mypy.ini +++ b/backend/mypy.ini @@ -51,3 +51,8 @@ ignore_missing_imports = true ignore_errors = true follow_imports = skip ignore_missing_imports = true + +[mypy-mistune.*] +ignore_errors = true +follow_imports = skip +ignore_missing_imports = true diff --git a/backend/nebula/config.py b/backend/nebula/config.py index 911a523..57d8041 100644 --- a/backend/nebula/config.py +++ b/backend/nebula/config.py @@ -11,7 +11,10 @@ class NebulaConfig(BaseModel): description="", ) - motd: str = Field("Nebula 6 ALPHA") + motd: str = Field( + "Nebula 6", + description="Message of the day", + ) postgres: PostgresDsn = Field( "postgres://nebula:nebula@postgres/nebula", diff --git a/backend/nebula/helpers/email.py b/backend/nebula/helpers/email.py index d157cfa..858a2b7 100644 --- a/backend/nebula/helpers/email.py +++ b/backend/nebula/helpers/email.py @@ -1,9 +1,9 @@ import smtplib -import nebula - from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +import nebula + try: import mistune # noqa @@ -37,19 +37,37 @@ def markdown2email(text) -> MIMEMultipart | MIMEText: return MIMEText(text, "plain") -def send_mail(to: str, subject: str, body: str | MIMEText | MIMEMultipart, **kwargs): - if type(to) == str: - to = [to] +def send_mail( + to: str | list[str], + subject: str, + body: str | MIMEText | MIMEMultipart, + **kwargs, +): + addresses: list[str] = [] + if isinstance(to, str): + addresses.append(to) + else: + addresses.extend(to) + reply_address = kwargs.get("from", nebula.settings.system.mail_from) - if isinstance(body, MIMEMultipart): - msg = body - else: + msg: MIMEText | MIMEMultipart + if isinstance(body, str): msg = MIMEText(body) + else: + msg = body msg["Subject"] = subject msg["From"] = reply_address msg["To"] = ",".join(to) + + try: + assert nebula.settings.system.smtp_host is not None, "SMTP host not set" + assert nebula.settings.system.smtp_port is not None, "SMTP port not set" + except AssertionError as e: + nebula.log.error(f"Unable to send email: {e}") + return + if nebula.settings.system.smtp_port == 25: s = smtplib.SMTP(nebula.settings.system.smtp_host, port=25) else: @@ -65,4 +83,4 @@ def send_mail(to: str, subject: str, body: str | MIMEText | MIMEMultipart, **kwa if user and password: s.login(user, password) - s.sendmail(reply_address, [to], msg.as_string()) + s.sendmail(reply_address, addresses, msg.as_string()) diff --git a/backend/server/__init__.py b/backend/server/__init__.py index be500d4..a101303 100644 --- a/backend/server/__init__.py +++ b/backend/server/__init__.py @@ -185,6 +185,7 @@ async def upload_media_file( async for chunk in request.stream(): i += len(chunk) await f.write(chunk) + nebula.log.debug(f"Uploaded {i} bytes", user=user.name) os.rename(temp_path, target_path) if direct: From aacc9670a8ca504d383cbf862e028e38b0fc2aef Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 7 Mar 2023 17:21:17 +0100 Subject: [PATCH 09/59] input switch fixed --- frontend/src/components/index.jsx | 4 +- frontend/src/components/switch.jsx | 79 +++++++++++-------- .../src/pages/assetEditor/assetEditorForm.jsx | 5 +- frontend/src/pages/services.jsx | 4 +- 4 files changed, 54 insertions(+), 38 deletions(-) diff --git a/frontend/src/components/index.jsx b/frontend/src/components/index.jsx index ba8ab7a..a913c66 100644 --- a/frontend/src/components/index.jsx +++ b/frontend/src/components/index.jsx @@ -3,7 +3,7 @@ import Dropdown from './dropdown' import Loader from './loader' import Progress from './progress' import Select from './select' -import Switch from './switch' +import InputSwitch from './switch' import Table from './table' import Video from './video' @@ -39,7 +39,7 @@ export { Progress, Select, Spacer, - Switch, + InputSwitch, Table, TextArea, Timestamp, diff --git a/frontend/src/components/switch.jsx b/frontend/src/components/switch.jsx index 5443f2b..e28ee2b 100644 --- a/frontend/src/components/switch.jsx +++ b/frontend/src/components/switch.jsx @@ -1,74 +1,87 @@ +/* import styled from 'styled-components' -import defaultTheme from './theme' const BaseSwitch = styled.div` max-height: ${(props) => props.theme.inputHeight}; min-height: ${(props) => props.theme.inputHeight}; +*/ +import styled from 'styled-components' +import defaultTheme from './theme' + +const BaseSwitch = ({ style, className, value, onChange }) => ( +
+ +
+) + +const InputSwitch = styled(BaseSwitch)` + max-height: ${(props) => props.theme.inputHeight}; + min-height: ${(props) => props.theme.inputHeight}; display: flex; flex-direction: row; align-items: center; - outline: 1px solid red; + justify-content: flex-start; - label { - --bheight: ${(props) => props.theme.inputHeight * 0.7}; - --bwidth: calc(var(--bheight) * 1.75 ); + --bheight: calc( ${(props) => props.theme.inputHeight} * 0.7); + + &.small { + --bheight: calc( ${(props) => props.theme.inputHeight} * 0.5); + min-height: 0; + } + --bwidth: calc(var(--bheight) * 1.75); + + .switch-body { position: relative; display: inline-block; height: var(--bheight); width: var(--bwidth); - input{ + input { opacity: 0; width: 0; height: 0; } - input:checked + .slider{ + input:checked + .slider { background-color: ${(props) => props.theme.colors.cyan}; } input:checked + .slider:before { - transform: translateX(calc(var(--bheight) * .8)); + transform: translateX(calc(var(--bheight) * 0.8)); } - span { + .slider { position: absolute; cursor: pointer; top: 0; left: 0; right: 0; bottom: 0; - background-color: ${(props) => props.theme.colors.surface05}; - transition: .4s; - border-radius: calc(var(--bheight) / 2): + background-color: ${(props) => props.theme.colors.surface04}; + transition: 0.4s; + border-radius: calc(var(--bheight) / 2); &:before { position: absolute; - content: ""; - height: calc(var(--bheight) * .8); - width: calc(var(--bheight) * .8); - left: calc(var(--bheight) * .1); - bottom: calc(var(--bheight) * .1); - background-color: ${(props) => props.theme.colors.surface03}; - transition: .4s; + content: ''; + height: calc(var(--bheight) * 0.8); + width: calc(var(--bheight) * 0.8); + left: calc(var(--bheight) * 0.1); + bottom: calc(var(--bheight) * 0.1); + background-color: ${(props) => props.theme.colors.textHl}; + transition: 0.4s; border-radius: 50%; + z-index: 1; } } - } + } ` -BaseSwitch.defaultProps = { - theme: defaultTheme, -} -const Switch = ({ style, className, ...props }) => { - return ( - - - - ) +InputSwitch.defaultProps = { + theme: defaultTheme, } -export default Switch +export default InputSwitch diff --git a/frontend/src/pages/assetEditor/assetEditorForm.jsx b/frontend/src/pages/assetEditor/assetEditorForm.jsx index 847cddb..8dd856b 100644 --- a/frontend/src/pages/assetEditor/assetEditorForm.jsx +++ b/frontend/src/pages/assetEditor/assetEditorForm.jsx @@ -3,7 +3,7 @@ import nebula from '/src/nebula' import { useMemo } from 'react' import { Form, FormRow, Select } from '/src/components' -import { InputText, TextArea, InputDatetime } from '/src/components/input' +import { InputText, TextArea, InputDatetime, InputSwitch } from '/src/components' const EditorField = ({ field, value, originalValue, onFieldChanged }) => { const metaType = { ...nebula.metaType(field.name), ...field } @@ -75,6 +75,9 @@ const EditorField = ({ field, value, originalValue, onFieldChanged }) => { case 'datetime': editor = break + case 'boolean': + editor = + break default: editor = } diff --git a/frontend/src/pages/services.jsx b/frontend/src/pages/services.jsx index 598d236..5333e9e 100644 --- a/frontend/src/pages/services.jsx +++ b/frontend/src/pages/services.jsx @@ -2,7 +2,7 @@ import nebula from '/src/nebula' import { useEffect, useState } from 'react' import { useDispatch } from 'react-redux' import PubSub from '/src/pubsub' -import { Table, Button, Switch } from '/src/components' +import { Table, Button, InputSwitch } from '/src/components' import { Duration } from 'luxon' import { setPageTitle } from '/src/actions' @@ -66,7 +66,7 @@ const ServicesPage = () => { return ( - + ) } From 56894714a972c677383914cd79fc39e6a9188efe Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 7 Mar 2023 18:07:12 +0100 Subject: [PATCH 10/59] integer editor, change password in the web ui, meta form sections, misc ui fixes --- backend/api/auth.py | 18 +-- backend/nebula/settings/models.py | 1 + backend/setup/defaults/folders.py | 9 +- frontend/src/components/form.jsx | 12 +- frontend/src/components/index.jsx | 2 + frontend/src/components/input.jsx | 12 ++ .../src/pages/assetEditor/assetEditorForm.jsx | 6 +- frontend/src/pages/mam.jsx | 5 + frontend/src/pages/profile.jsx | 110 ++++++++++++------ 9 files changed, 126 insertions(+), 49 deletions(-) diff --git a/backend/api/auth.py b/backend/api/auth.py index 925d006..2a4d296 100644 --- a/backend/api/auth.py +++ b/backend/api/auth.py @@ -1,8 +1,8 @@ -from fastapi import Depends, Header +from fastapi import Depends, Header, Response from pydantic import Field import nebula -from nebula.exceptions import NotFoundException, UnauthorizedException + from server.dependencies import current_user from server.models import RequestModel, ResponseModel from server.request import APIRequest @@ -68,15 +68,15 @@ class LogoutRequest(APIRequest): async def handle(self, authorization: str | None = Header(None)): if not authorization: - raise UnauthorizedException("No authorization header provided") + raise nebula.UnauthorizedException("No authorization header provided") access_token = parse_access_token(authorization) if not access_token: - raise UnauthorizedException("Invalid authorization header provided") + raise nebula.UnauthorizedException("Invalid authorization header provided") await Session.delete(access_token) - raise UnauthorizedException("Logged out") + raise nebula.UnauthorizedException("Logged out") class SetPassword(APIRequest): @@ -95,7 +95,7 @@ async def handle( ): if request.login: if not user.is_admin: - raise UnauthorizedException( + raise nebula.UnauthorizedException( "Only admin can change other user's password" ) query = "SELECT meta FROM users WHERE login = $1" @@ -103,14 +103,14 @@ async def handle( target_user = nebula.User.from_row(row) break else: - raise NotFoundException(f"User {request.login} not found") + raise nebula.NotFoundException(f"User {request.login} not found") else: target_user = user if len(request.password) < 8: - raise UnauthorizedException("Password is too short") + raise nebula.BadRequestException("Password is too short") target_user.set_password(request.password) await user.save() - return ResponseModel() + return Response(status_code=204) diff --git a/backend/nebula/settings/models.py b/backend/nebula/settings/models.py index 62db255..5a365ce 100644 --- a/backend/nebula/settings/models.py +++ b/backend/nebula/settings/models.py @@ -178,6 +178,7 @@ class StorageSettings(BaseStorageSettings): class FolderField(SettingsModel): name: str = Field(..., title="Field name") + section: str | None = Field(None, title="Section") mode: str | None = None format: str | None = None order: str | None = None diff --git a/backend/setup/defaults/folders.py b/backend/setup/defaults/folders.py index d9fb3e0..acd6952 100644 --- a/backend/setup/defaults/folders.py +++ b/backend/setup/defaults/folders.py @@ -14,18 +14,18 @@ ] serie_description: FieldList = [ - FolderField(name="serie"), + FolderField(name="serie", section="Series"), FolderField(name="serie/season"), FolderField(name="serie/episode"), ] roles_description: FieldList = [ - FolderField(name="role/director"), + FolderField(name="role/director", section="Roles"), FolderField(name="role/cast"), ] content_description: FieldList = [ - FolderField(name="genre", filter=movie_genre_pattern), + FolderField(name="genre", filter=movie_genre_pattern, section="Content"), FolderField(name="editorial_format", filter=r"^2(\.\d+){0,2}$"), FolderField(name="atmosphere"), FolderField(name="intention", filter=r"^1\.(1|2|3|4|5|6|7|8)$"), @@ -35,11 +35,12 @@ ] production_description: FieldList = [ - FolderField(name="date/valid"), + FolderField(name="date/valid", section="Production"), FolderField(name="editorial_control"), FolderField(name="rights"), FolderField(name="rights/type"), FolderField(name="rights/description"), + FolderField(name="rights/ott"), FolderField(name="notes"), FolderField(name="qc/report"), ] diff --git a/frontend/src/components/form.jsx b/frontend/src/components/form.jsx index fcb1f44..4b0d435 100644 --- a/frontend/src/components/form.jsx +++ b/frontend/src/components/form.jsx @@ -5,6 +5,13 @@ const Form = styled.div` flex-direction: column; gap: var(--gap-size); + h3 { + width: 100%; + text-align: center; + font-size: 1rem; + font-weight: 500; + } + .form-row { display: flex; flex-direction: row; @@ -27,14 +34,17 @@ const Form = styled.div` } ` -const FormRow = ({ title, tooltip, children, ...props }) => { +const FormRow = ({ title, tooltip, section, children, ...props }) => { return ( + <> + {section &&

{section}

}
{title}
{children}
+ ) } diff --git a/frontend/src/components/index.jsx b/frontend/src/components/index.jsx index a913c66..9d1bbb5 100644 --- a/frontend/src/components/index.jsx +++ b/frontend/src/components/index.jsx @@ -9,6 +9,7 @@ import Video from './video' import { InputDatetime, + InputInteger, InputNumber, InputPassword, InputText, @@ -30,6 +31,7 @@ export { Form, FormRow, InputDatetime, + InputInteger, InputNumber, InputPassword, InputText, diff --git a/frontend/src/components/input.jsx b/frontend/src/components/input.jsx index 507a1fb..ac2f657 100644 --- a/frontend/src/components/input.jsx +++ b/frontend/src/components/input.jsx @@ -182,6 +182,17 @@ const InputPassword = ({ value, onChange, ...props }) => { ) } +const InputInteger = ({ value, onChange, ...props }) => { + return ( + onChange(e.target.value)} + {...props} + /> + ) +} + const TextArea = ({ value, onChange, ...props }) => { return ( { const metaType = { ...nebula.metaType(field.name), ...field } @@ -78,6 +78,9 @@ const EditorField = ({ field, value, originalValue, onFieldChanged }) => { case 'boolean': editor = break + case 'integer': + editor = + break default: editor = } @@ -86,6 +89,7 @@ const EditorField = ({ field, value, originalValue, onFieldChanged }) => { {editor} diff --git a/frontend/src/pages/mam.jsx b/frontend/src/pages/mam.jsx index 4415c52..7a76f61 100644 --- a/frontend/src/pages/mam.jsx +++ b/frontend/src/pages/mam.jsx @@ -23,6 +23,11 @@ const MAMContainer = styled.div` gap: 8px; min-width: 400px; } + + .__dbk__child-wrapper:last-child { + min-width: 1000px; + } + ` const MAMPage = () => { diff --git a/frontend/src/pages/profile.jsx b/frontend/src/pages/profile.jsx index 8755e3e..ab5e17e 100644 --- a/frontend/src/pages/profile.jsx +++ b/frontend/src/pages/profile.jsx @@ -1,45 +1,85 @@ import nebula from '/src/nebula' +import { toast } from 'react-toastify' +import { useEffect, useState } from 'react' import { Form, FormRow, InputText, Button } from '/src/components' -const ProfilePage = () => { + +const ProfileForm = () => { const displayName = nebula.user.full_name || nebula.user.login + return ( +
+

Profile: {displayName}

+ +
+ + + + + + + + + + +
+ ) +} + + +const ChangePasswordForm = () => { + const [password, setPassword] = useState('') + const [passwordRepeat, setPasswordRepeat] = useState('') + + const changePassword = () => { + if (password !== passwordRepeat) { + toast.error('Passwords do not match') + return + } + + nebula + .request('password', { password }) + .then(() => { + toast.success('Password changed') + setPassword('') + setPasswordRepeat('') + }) + .catch((err) => { + const msg = err.response?.data?.detail || err.message + toast.error(msg) + }) + } + + return ( +
+

Change password

+
+ + + + + + + +
+ ) +} + + + +const ProfilePage = () => { + return (
-
-

Profile: {displayName}

- -
- - - - - - - - - - -
- -
-

Change password

-
- - - - - - - -
+ +

Access control

@@ -48,9 +88,11 @@ const ProfilePage = () => { )}
+ {/*

Active sessions

+ */}
) } From 6cc237b9c7bb9923b75843f9d82569e51ea63511 Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 7 Mar 2023 20:54:25 +0100 Subject: [PATCH 11/59] handle allow-if in actions endpoint --- backend/api/jobs/actions.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/api/jobs/actions.py b/backend/api/jobs/actions.py index 7e252fc..cba8ade 100644 --- a/backend/api/jobs/actions.py +++ b/backend/api/jobs/actions.py @@ -1,5 +1,6 @@ from fastapi import Depends from pydantic import Field +from nxtools import xml import nebula from server.dependencies import current_user @@ -46,14 +47,23 @@ async def handle( """ async for row in nebula.db.iterate(query): - # TODO: implement allow-if and ACL - result.append( - ActionItemModel( - id=row["id"], - name=row["title"], - ) - ) - + # TODO: implement ACL + action_settings = xml(row["settings"]) + + if allow_if_elm := action_settings.findall("allow_if"): + allow_if_cond = allow_if_elm[0].text + + for id_asset in request.ids: + asset = await nebula.Asset.load(id_asset) + assert asset + if not eval(allow_if_cond): + break + else: + result.append( + ActionItemModel( + id=row["id"], + name=row["title"], + ) + ) nebula.log.info(f"Actions for assets {request.ids} are {result}") - return ActionsResponseModel(actions=result) From 4922f9ac56d945f67f51dcbb9b55d398f7a0e1dc Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 7 Mar 2023 21:50:26 +0100 Subject: [PATCH 12/59] more acl --- backend/api/delete.py | 4 ++-- backend/api/jobs/actions.py | 16 ++++++++++++---- backend/api/rundown/__init__.py | 7 ++----- backend/api/scheduler/__init__.py | 6 ++++-- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/backend/api/delete.py b/backend/api/delete.py index 283e075..7ce5d95 100644 --- a/backend/api/delete.py +++ b/backend/api/delete.py @@ -58,10 +58,10 @@ async def handle( ) case ObjectType.ASSET | ObjectType.EVENT: - # TODO: ACL HERE + # TODO: ACL HERE? # In general, normal users don't need to # delete assets or events directly - if not user["is_admin"]: + if not user.is_admin: raise nebula.ForbiddenException( "You are not allowed to delete this object" ) diff --git a/backend/api/jobs/actions.py b/backend/api/jobs/actions.py index cba8ade..5a00e6b 100644 --- a/backend/api/jobs/actions.py +++ b/backend/api/jobs/actions.py @@ -9,12 +9,17 @@ class ActionsRequestModel(RequestModel): - ids: list[int] + ids: list[int] = Field( + ..., + title="Asset IDs", + description="List of asset IDs for which to get available actions", + example=[1, 2, 3], + ) class ActionItemModel(ResponseModel): - id: int = Field(..., title="Action ID") - name: str = Field(..., title="Action name") + id: int = Field(..., title="Action ID", example=1) + name: str = Field(..., title="Action name", example="proxy") class ActionsResponseModel(ResponseModel): @@ -47,7 +52,10 @@ async def handle( """ async for row in nebula.db.iterate(query): - # TODO: implement ACL + + if not user.can("job_control", row["id"]): + continue + action_settings = xml(row["settings"]) if allow_if_elm := action_settings.findall("allow_if"): diff --git a/backend/api/rundown/__init__.py b/backend/api/rundown/__init__.py index 48437b6..327b946 100644 --- a/backend/api/rundown/__init__.py +++ b/backend/api/rundown/__init__.py @@ -21,10 +21,7 @@ async def handle( user: nebula.User = Depends(current_user), ) -> RundownResponseModel: - # TODO: Handle ACL here - # if not ( - # user.has_right("rundown_view", id_channel) - # or user.has_right("rundown_edit", id_channel) - # ): + if user.can("rundown_view", request.id_channel): + raise nebula.ForbiddenException("You are not allowed to view this rundown") return await get_rundown(request) diff --git a/backend/api/scheduler/__init__.py b/backend/api/scheduler/__init__.py index ce0aafe..a5b7d3e 100644 --- a/backend/api/scheduler/__init__.py +++ b/backend/api/scheduler/__init__.py @@ -22,9 +22,11 @@ async def handle( initiator: str = Depends(request_initiator), ) -> SchedulerResponseModel: - has_rights = True # TODO + if user.can("scheduler_view", request.id_channel): + raise nebula.ForbiddenException("You are not allowed to view this channel") - result = await scheduler(request, has_rights) + editable = user.can("scheduler_edit", request.id_channel) + result = await scheduler(request, editable) if result.affected_events: await nebula.msg( From 0d1fd55097c62c2170f3a84378f2b43fc43b9a2b Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 8 Mar 2023 09:02:51 +0100 Subject: [PATCH 13/59] allow appending items to new events in schedule query --- backend/api/rundown/__init__.py | 2 +- backend/api/scheduler/__init__.py | 2 +- backend/api/scheduler/models.py | 2 ++ backend/api/scheduler/scheduler.py | 16 +++++++++++++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/backend/api/rundown/__init__.py b/backend/api/rundown/__init__.py index 327b946..db22340 100644 --- a/backend/api/rundown/__init__.py +++ b/backend/api/rundown/__init__.py @@ -21,7 +21,7 @@ async def handle( user: nebula.User = Depends(current_user), ) -> RundownResponseModel: - if user.can("rundown_view", request.id_channel): + if not user.can("rundown_view", request.id_channel): raise nebula.ForbiddenException("You are not allowed to view this rundown") return await get_rundown(request) diff --git a/backend/api/scheduler/__init__.py b/backend/api/scheduler/__init__.py index a5b7d3e..52d4c37 100644 --- a/backend/api/scheduler/__init__.py +++ b/backend/api/scheduler/__init__.py @@ -22,7 +22,7 @@ async def handle( initiator: str = Depends(request_initiator), ) -> SchedulerResponseModel: - if user.can("scheduler_view", request.id_channel): + if not user.can("scheduler_view", request.id_channel): raise nebula.ForbiddenException("You are not allowed to view this channel") editable = user.can("scheduler_edit", request.id_channel) diff --git a/backend/api/scheduler/models.py b/backend/api/scheduler/models.py index c4f7441..0f2d90f 100644 --- a/backend/api/scheduler/models.py +++ b/backend/api/scheduler/models.py @@ -26,6 +26,8 @@ class EventData(RequestModel): example=123, ) + items: list[dict[str, Serializable]] | None = Field(default_factory=list) + meta: dict[str, Serializable] | None = Field( default=None, title="Event metadata", diff --git a/backend/api/scheduler/scheduler.py b/backend/api/scheduler/scheduler.py index 3e432c4..58118fe 100644 --- a/backend/api/scheduler/scheduler.py +++ b/backend/api/scheduler/scheduler.py @@ -26,6 +26,7 @@ async def create_new_event( new_event["start"] = event_data.start asset_meta = {} + position = 0 if event_data.id_asset: asset = await nebula.Asset.load(event_data.id_asset, connection=conn) @@ -35,13 +36,26 @@ async def create_new_event( new_item = nebula.Item(connection=conn) new_item["id_asset"] = event_data.id_asset new_item["id_bin"] = new_bin.id - new_item["position"] = 0 + new_item["position"] = position new_item["mark_in"] = asset["mark_in"] new_item["mark_out"] = asset["mark_out"] await new_item.save() new_bin["duration"] = asset.duration asset_meta = asset.meta + position += 1 + + if event_data.items: + for item_data in event_data.items: + if item_data.get("id"): + item = await nebula.Item.load(item_data["id"], connection=conn) + else: + item = nebula.Item(connection=conn) + item.update(item_data) + item["id_bin"] = new_bin.id + item["position"] = position + await item.save() + position += 1 for field in channel.fields: if (value := asset_meta.get(field.name)) is not None: From 07272b7dfbb87a1c78191476db5969a1dec14fad Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 8 Mar 2023 21:38:16 +0100 Subject: [PATCH 14/59] Added more default meta types --- backend/schema/meta-aliases-cs.json | 2 ++ backend/schema/meta-aliases-en.json | 2 ++ backend/setup/defaults/meta_types.py | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/backend/schema/meta-aliases-cs.json b/backend/schema/meta-aliases-cs.json index b307cbb..69cdb62 100644 --- a/backend/schema/meta-aliases-cs.json +++ b/backend/schema/meta-aliases-cs.json @@ -1,5 +1,6 @@ [ ["commercial/content" , "Reklamí obsah" , "Rekl. obsah" , ""], + ["commercial/pp" , "Product placement" , "PP" , "Obsahuje product placement"], ["solver" , "Solver" , null , ""], ["year" , "Rok" , null , "Rok vydání díla"], ["id_user" , "ID uživatele" , null , ""], @@ -75,6 +76,7 @@ ["audio/silence" , "Ticho" , null , ""], ["place" , "Místo" , null , ""], ["place/type" , "Typ místa" , null , ""], + ["place/specific" , "Upřesnění místa" , null , ""], ["video/is_interlaced" , "Prokládání" , null , ""], ["id_item" , "Item ID" , null , ""], ["rights/spatial" , "Regionální omezení" , null , ""], diff --git a/backend/schema/meta-aliases-en.json b/backend/schema/meta-aliases-en.json index 10fcbbb..9734cf3 100644 --- a/backend/schema/meta-aliases-en.json +++ b/backend/schema/meta-aliases-en.json @@ -1,5 +1,6 @@ [ ["commercial/content" , "Commercial content" , "Content" , ""], + ["commercial/pp" , "Product placement" , "PP" , "Contains product placement"], ["solver" , "Solver" , null , ""], ["year" , "Year" , null , ""], ["id_user" , "User ID" , null , ""], @@ -75,6 +76,7 @@ ["audio/silence" , "Silence" , null , ""], ["place" , "Place" , null , ""], ["place/type" , "Place type" , null , ""], + ["place/specific" , "Place specification" , null , ""], ["video/is_interlaced" , "Is interlaced" , "Interlaced" , ""], ["id_item" , "Item ID" , null , ""], ["rights/spatial" , "Spatial rights" , null , ""], diff --git a/backend/setup/defaults/meta_types.py b/backend/setup/defaults/meta_types.py index ff8302f..d90d0cf 100644 --- a/backend/setup/defaults/meta_types.py +++ b/backend/setup/defaults/meta_types.py @@ -396,6 +396,11 @@ "type": T.SELECT, "cs": "urn:site:clients", }, + "commercial/pp": { + "ns": "m", + "type": T.SELECT, + "cs": "urn:site:clients", + }, "runs/daily": { "ns": "m", "type": T.INTEGER, From ef45651eccb677e6b31cfdc5a85674994fed4f81 Mon Sep 17 00:00:00 2001 From: Martastain Date: Wed, 8 Mar 2023 22:47:37 +0100 Subject: [PATCH 15/59] feat: assignees --- backend/api/browse.py | 4 +- backend/api/get.py | 32 ++++++++++++--- frontend/src/components/select.jsx | 1 + .../src/pages/assetEditor/assetEditorNav.jsx | 5 +++ frontend/src/pages/assetEditor/assignees.jsx | 41 +++++++++++++++++++ 5 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 frontend/src/pages/assetEditor/assignees.jsx diff --git a/backend/api/browse.py b/backend/api/browse.py index 0eca961..6658f61 100644 --- a/backend/api/browse.py +++ b/backend/api/browse.py @@ -214,7 +214,9 @@ def build_query( # Access control if user.is_limited: - cond_list.append(f"meta->>'created_by' = '{user.id}'") + c1 = f"meta->>'created_by' = '{user.id}'" + c2 = f"meta->'assignees' @> '[{user.id}]'::JSONB" + cond_list.append(f"({c1} OR {c2})") # Build conditions diff --git a/backend/api/get.py b/backend/api/get.py index 7228713..3c423f8 100644 --- a/backend/api/get.py +++ b/backend/api/get.py @@ -38,6 +38,28 @@ class GetResponseModel(ResponseModel): ) +def can_access_object(user: nebula.User, meta: dict[str, Any]) -> False: + if user.is_admin: + return True + elif user.id in meta.get("assignees", []): + return True + elif user.is_limited: + if meta.get("created_by") != user.id: + return False + return True + if id_folder := meta.get("id_folder"): + # Users can view assets in folders they have access to + return user.can("asset_view", id_folder) + + if login := meta.get("login"): + # Users can view their own data + return login == user.name + + # Normal users don't need to access items, bins or events + # using get requests. + return False + + class Request(APIRequest): """Get a list of objects""" @@ -56,12 +78,10 @@ async def handle( data = [] async for row in nebula.db.iterate(query, request.ids): - if user.is_limited: - # Limited users can only see their own objects - if row["meta"].get("created_by") != user.id: - raise nebula.ForbiddenException( - "You are not allowed to access this object" - ) + if not can_access_object(user, row["meta"]): + raise nebula.ForbiddenException( + "You are not allowed to access this object" + ) data.append(row["meta"]) return GetResponseModel(data=data) diff --git a/frontend/src/components/select.jsx b/frontend/src/components/select.jsx index 445af1b..bdd2887 100644 --- a/frontend/src/components/select.jsx +++ b/frontend/src/components/select.jsx @@ -326,4 +326,5 @@ const Select = ({ ) } +export {SelectDialog} export default Select diff --git a/frontend/src/pages/assetEditor/assetEditorNav.jsx b/frontend/src/pages/assetEditor/assetEditorNav.jsx index 0cd6db2..0f02c2f 100644 --- a/frontend/src/pages/assetEditor/assetEditorNav.jsx +++ b/frontend/src/pages/assetEditor/assetEditorNav.jsx @@ -17,6 +17,7 @@ import SendToDialog from '/src/containers/sendTo' import MetadataDetail from './detail' import ContextActionResult from './contextAction' import UploadDialog from './uploadDialog' +import Assignees from './assignees' import contentType from 'content-type' @@ -184,6 +185,10 @@ const AssetEditorNav = ({ options={assetActions} disabled={!assetData?.id || isChanged} /> + setMeta('assignees', val)} + /> )} diff --git a/frontend/src/pages/assetEditor/assignees.jsx b/frontend/src/pages/assetEditor/assignees.jsx new file mode 100644 index 0000000..450c670 --- /dev/null +++ b/frontend/src/pages/assetEditor/assignees.jsx @@ -0,0 +1,41 @@ +import nebula from '/src/nebula' +import {useMemo, useState} from 'react' +import { Button } from '/src/components' +import {SelectDialog} from '/src/components/select' + +const Assignees = ({ assignees, setAssignees }) => { + const [dialogVisible, setDialogVisible] = useState(false) + + const options = useMemo(() => { + return nebula.settings.users.map((user) => { + return { + value: `${user.id}`, + title: user.name, + } + }) + }, [nebula.settings.users]) + + return ( + <> + {dialogVisible && ( + { + setAssignees(value.map((v) => parseInt(v))) + setDialogVisible(false) + }} + /> + )} +