diff --git a/.ci/deployment/badges.py b/.ci/deployment/badges.py new file mode 100644 index 000000000000..95c36fa5f40f --- /dev/null +++ b/.ci/deployment/badges.py @@ -0,0 +1,138 @@ +import os +import json +import time + + +def setup_git(token): + # Set git credentials + os.environ['GIT_ASKPASS'] = 'echo' + os.environ['GIT_USERNAME'] = 'github-actions[bot]' + os.environ['GIT_PASSWORD'] = token + os.environ['GIT_EMAIL'] = 'github-actions[bot]@users.noreply.github.com' + os.environ['GIT_COMMITTER_NAME'] = 'github-actions[bot]' + os.environ['GIT_COMMITTER_EMAIL'] = 'github-actions[bot]@users.noreply.github.com' + + +def setup_branch(org, repo, branch): + if os.path.isdir(repo): + os.system(f"rm -rf {repo}") + + # Clear repo directory except .git folder + os.system( + f"git clone --depth 1 -b {branch} https://{os.environ['GIT_PASSWORD']}@github.com/{org}/{repo} && " + f"cd {repo} && " + "find . -not -path './.git/*' -delete" + ) + # Create empty shield.json + os.system( + f"cd {repo} && " + "echo '{}' > shields.json" + ) + # Create branch, commit and push + os.system( + f"cd {repo} && " + f"git checkout --orphan {branch} && " + f"git add . && " + f"git commit --author='github-actions[bot] ' -m 'Create {branch}' && " + f"git push --quiet origin {branch}" + ) + + +# Pull branch from GitHub repository using git command +def pull_branch(org, repo, branch): + # Remove repository if exists + if os.path.isdir(repo): + os.system(f"rm -rf {repo}") + + # Clone repository + code = os.system( + f"git clone --depth 1 -b {branch} https://{os.environ['GIT_PASSWORD']}@github.com/{org}/{repo} && " + f"cd {repo} && " + f"git pull origin {branch}" + ) + + if code == 256: + # Branch does not exist + setup_branch(org, repo, branch) + return code == 0 or code == 256 + + +# Push branch to GitHub repository using git command +def commit_and_push_branch(repo, branch): + author_name = os.environ['GIT_COMMITTER_NAME'] + author_email = os.environ['GIT_COMMITTER_EMAIL'] + code = os.system( + f"cd {repo} && " + f"git add . && " + f"git commit --author='{author_name} <{author_email}>' -m 'Update {branch}' && " + f"git push --quiet origin {branch}" + ) + return code == 0 or code == 256 + + +def remove_repo(repo): + code = os.system( + f"rm -rf {repo}" + ) + return code == 0 + + +def read_badges_json(repo, file_name="shields.json"): + with open(f"{repo}/{file_name}") as f: + return json.load(f) + + +def write_badges_json(badges, repo, file_name="shields.json"): + with open(f"{repo}/{file_name}", "w") as f: + json.dump(badges, f, indent=4) + + +def set_badge_values(repo, badge_name, badge_status, badge_color, lifetime=None): + badges = read_badges_json(repo) + if badge_name not in badges: + badges[badge_name] = {} + badges[badge_name]["status"] = badge_status + badges[badge_name]["color"] = badge_color + badges[badge_name]["timestamp"] = int(time.time()) + badges[badge_name]["lifetime"] = lifetime + write_badges_json(badges, repo) + + +def set_badge(token, org, repo, badge_name, badge_status, badge_color, branch="shields"): + setup_git(token) + pull_branch(org, repo, branch) + set_badge_values(repo, badge_name, badge_status, badge_color) + commit_and_push_branch(repo, branch) + remove_repo(repo) + + +def get_badge_status(token, org, repo, badge_name, branch="shields"): + setup_git(token) + pull_branch(org, repo, branch) + badges = read_badges_json(repo) + remove_repo(repo) + if badge_name not in badges: + return True + current_time = int(time.time()) + badge_status = badges[badge_name]["status"] + badge_color = badges[badge_name]["color"] + badge_time = badges[badge_name]["timestamp"] + badge_lifetime = badges[badge_name]["lifetime"] + if (badge_time is None or badge_lifetime is None) and badge_color == "red": + return False + return current_time - badge_time > badge_lifetime or badge_color != "red", badge_status + + +def unlock_badges(token, org, repo, branch="shields"): + setup_git(token) + pull_branch(org, repo, branch) + badges = read_badges_json(repo) + current_time = int(time.time()) + for badge_name in badges: + badge_time = badges[badge_name]["timestamp"] + badge_lifetime = badges[badge_name]["lifetime"] + if badge_lifetime is None: + continue + if current_time - badge_time > badge_lifetime: + badges[badge_name]["color"] = "green" + write_badges_json(badges, repo) diff --git a/.ci/deployment/github_api.py b/.ci/deployment/github_api.py new file mode 100644 index 000000000000..6d4c1c59946d --- /dev/null +++ b/.ci/deployment/github_api.py @@ -0,0 +1,203 @@ +from enum import Enum + +import requests + +GITHUB_TOKEN = None +GITHUB_ORG_TOKEN = None +org = None +repo = None + + +class DeploymentStatus(Enum): + ERROR = "error" + FAILURE = "failure" + INACTIVE = "inactive" + IN_PROGRESS = "in_progress" + QUEUED = "queued" + PENDING = "pending" + SUCCESS = "success" + + +# Check if user is in GitHub group using GitHub API +def is_user_in_github_group(user, group): + url = f"https://api.github.com/orgs/{org}/teams/{group}/memberships/{user}" + headers = {"Authorization": f"token {GITHUB_ORG_TOKEN}"} + response = requests.get(url, headers=headers) + return response.status_code == 200 + + +# List all teams in GitHub organization +def list_groups(): + url = f"https://api.github.com/orgs/{org}/teams" + headers = {"Authorization": f"token {GITHUB_ORG_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# List all environments in GitHub repository +def list_environments(): + url = f"https://api.github.com/repos/{org}/{repo}/environments" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# Get environment by name +def get_environment(name): + url = f"https://api.github.com/repos/{org}/{repo}/environments/{name}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# List all deployments in GitHub repository +def list_deployments(): + url = f"https://api.github.com/repos/{org}/{repo}/deployments" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# List deployments for environment +def list_deployments_for_environment(environment): + url = f"https://api.github.com/repos/{org}/{repo}/deployments?environment={environment}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# Get deployment by id +def get_deployment(id): + url = f"https://api.github.com/repos/{org}/{repo}/deployments/{id}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +# Get sha for branch +def get_sha_for_branch(branch): + url = f"https://api.github.com/repos/{org}/{repo}/branches/{branch}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()["commit"]["sha"] + elif response.status_code == 404: + return None + + +# Get sha for tag +def get_sha_for_tag(tag): + url = f"https://api.github.com/repos/{org}/{repo}/git/ref/tags/{tag}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()["object"]["sha"] + elif response.status_code == 404: + return None + + +# Get sha for ref +def get_sha_for_ref(ref): + url = f"https://api.github.com/repos/{org}/{repo}/git/ref/{ref}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()["object"]["sha"] + elif response.status_code == 404: + return None + + +# Get sha for branch/tag/ref +def get_sha(ref): + sha = get_sha_for_branch(ref) + if sha is None: + sha = get_sha_for_tag(ref) + if sha is None: + sha = get_sha_for_ref(ref) + if sha is None: + sha = ref + return sha + + +# Create deployment for environment +def create_deployment(environment, branch): + url = f"https://api.github.com/repos/{org}/{repo}/deployments" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + data = { + "ref": branch, + "sha": get_sha_for_branch(branch), + "environment": environment, + "required_contexts": [], + "auto_merge": False, + "transient_environment": False, + "production_environment": False, + } + response = requests.post(url, headers=headers, json=data) + return response.json() + + +# Create deployment status for deployment +def create_deployment_status(deployment, environment_url, state: DeploymentStatus, description=""): + url = f"https://api.github.com/repos/{org}/{repo}/deployments/{deployment}/statuses" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + data = { + "state": state.value, + "description": description, + "environment_url": environment_url, + "auto_inactive": False, + } + response = requests.post(url, headers=headers, json=data) + return response.json() + + +# Create a comment on a pull request +def create_comment(pr, comment): + url = f"https://api.github.com/repos/{org}/{repo}/issues/{pr}/comments" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + data = {"body": comment} + response = requests.post(url, headers=headers, json=data) + return response.json() + + +def add_label(pr, label): + url = f"https://api.github.com/repos/{org}/{repo}/issues/{pr}/labels" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + data = [label] + response = requests.post(url, headers=headers, json=data) + return response.json() + + +def remove_label(pr, label): + url = f"https://api.github.com/repos/{org}/{repo}/issues/{pr}/labels/{label}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.delete(url, headers=headers) + return response.json() + + +def check_build_job(sha, workflow="build.yml"): + url = f"https://api.github.com/repos/{org}/{repo}/actions/workflows/{workflow}/runs?status=success&head_sha={sha}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json()["total_count"] > 0 + + +# const opts = github.rest.issues.listForRepo.endpoint.merge({ +# owner: context.repo.owner, +# repo: context.repo.repo, +# labels: ['lock:${{ matrix.label-identifier }}'] +# }) +def get_issues_with_label(label): + url = f"https://api.github.com/repos/{org}/{repo}/issues?labels={label}" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + return response.json() + + +def get_pr_for_ref(ref): + url = f"https://api.github.com/repos/{org}/{repo}/commits/{ref}/pulls" + headers = {"Authorization": f"token {GITHUB_TOKEN}"} + response = requests.get(url, headers=headers) + if response.status_code == 200: + return response.json()[0]["number"] + elif response.status_code == 404: + return None diff --git a/.ci/deployment/main.py b/.ci/deployment/main.py new file mode 100644 index 000000000000..114529b0431e --- /dev/null +++ b/.ci/deployment/main.py @@ -0,0 +1,150 @@ +import os +import sys + +from badges import set_badge, get_badge_status +import github_api + + +def fail(pr, exit_code, message): + print(f"::error::{message}", file=sys.stderr) + if pr: + print(f"Creating comment on PR {pr}") + github_api.create_comment(pr, f"#### ⚠️ Unable to deploy to environment ⚠️\n{message}") + sys.exit(exit_code) + + +def setup_ssh(): + ssh_auth_sock = os.getenv("SSH_AUTH_SOCK") + gateway_ssh_key = os.getenv("GATEWAY_SSH_KEY") + deployment_ssh_key = os.getenv("DEPLOYMENT_SSH_KEY") + gateway_host_public_key = os.getenv("GATEWAY_HOST_PUBLIC_KEY") + deployment_host_public_keys = os.getenv("DEPLOYMENT_HOST_PUBLIC_KEYS") + + os.system( + f"mkdir -p ~/.ssh && " + f"ssh-agent -a {ssh_auth_sock} > /dev/null && " + f"ssh-add - <<< {gateway_ssh_key} && " + f"ssh-add - <<< {deployment_ssh_key} && " + f"cat - <<< {gateway_host_public_key} >> ~/.ssh/known_hosts && " + f"cat - <<< $(sed 's/\\n/\n/g' <<< \"{deployment_host_public_keys}\") >> ~/.ssh/known_hosts" + ) + + +def check_lock_status(pr, label): + issues = github_api.get_issues_with_label(f"lock:{label}") + if issues.__len__() == 1 and issues[0]["number"] != pr: + fail(pr, 400, f"Environment {label} is already in use by PR #{issues[0].number}.") + elif issues.__len__() > 1: + fail(pr, 400, f'Environment {label} is already in use by multiple PRs. Check PRs with label "lock:{label}"!') + + +def deploy(ref): + setup_ssh() + deployment_hosts = os.getenv("DEPLOYMENT_HOSTS").split(" ") + deployment_user = os.getenv("DEPLOYMENT_USER") + gateway_user = os.getenv("GATEWAY_USER") + gateway_host = os.getenv("GATEWAY_HOST") + deployment_folder = os.getenv("DEPLOYMENT_FOLDER") + script_base = os.getenv("SCRIPT_PATH", "./artemis-server-cli docker-deploy") + + tag = os.getenv("TAG") + + for host in deployment_hosts: + code = os.system(f"{script_base} \"{deployment_user}@{host}\" -g \"{gateway_user}@{gateway_host}\" -t {tag} -b {ref} -d {deployment_folder} -y") + if code != 0: + fail(os.getenv("PR"), 500, f"Deployment to {host} failed during ssh script execution. Check the logs for more information.") + + +def __main__(): + print("Reading environment variables") + # Get github token from environment variable + github_token = os.getenv("GITHUB_TOKEN") + # Get org github token from environment variable + github_org_token = os.getenv("GITHUB_ORG_TOKEN", github_token) + # Get user from environment variable + user = os.getenv("GITHUB_USER") + # Get org from environment variable + org = os.getenv("ORG", "ls1intum") + # Get repo from environment variable + repo = os.getenv("REPO", "Artemis") + # Get group from environment variable + group = os.getenv("GROUP", "artemis-developers") + # Get ref from environment variable + ref = os.getenv("REF", "develop") + # Get PR from environment variable + pr = os.getenv("PR", None) + # Get test server from environment variable + label = os.getenv("LABEL") + # Get badge from environment variable + badge = os.getenv("BADGE", label) + + # Get environment url from env + environment_url = os.getenv("ENVIRONMENT_URL", f"https://{label}.artemis.cit.tum.de") + # Get environment name from env + environment_name = os.getenv("ENVIRONMENT_NAME", environment_url) + # Get environment management from env + manage_environment = os.getenv("MANAGE_ENVIRONMENT", "false").lower() == "true" + + github_api.GITHUB_TOKEN = github_token + github_api.GITHUB_ORG_TOKEN = github_org_token + github_api.org = org + github_api.repo = repo + + # Attempt to get PR from ref + if not pr: + pr = github_api.get_pr_for_ref(ref) + + # Remove deployment label from PR + if pr: + print(f"Removing label {label} from PR {pr} (if exists)") + github_api.remove_label(pr, f"deploy:{label}") + + print(f"Checking if user {user} is in GitHub group {group}") + # Check if user is in GitHub group + if not github_api.is_user_in_github_group(user, group): + fail(pr, 403, f"User {user} does not have access to deploy to {label}.") + + print(f"Setting sha for ref {ref}") + # Get sha if ref is not a sha + sha = github_api.get_sha(ref) + print("SHA:", sha) + print(f"Checking if build job for sha ran successfully") + # Check that build job ran successfully + if not github_api.check_build_job(sha): + fail(pr, 400, f"The docker build needs to run through before deploying.") + + print(f"Checking if environment {label} is available") + check_lock_status(pr, label) + + available, badge_status = get_badge_status(github_token, org, repo, badge) + if not available: + fail(pr, 400, f"Environment {label} is already in use by ´{badge_status}´.") + + if manage_environment: + print(f"Creating deployment for {environment_name}") + deployment = github_api.create_deployment(environment_name, sha) + print(f"Creating deployment status for {environment_name}") + github_api.create_deployment_status(deployment["id"], environment_url, github_api.DeploymentStatus.IN_PROGRESS) + + try: + print("Deploying") + deploy(ref) + + print(f"Deployment to {label} successful.") + if manage_environment: + print(f"Updating deployment status for {environment_name}") + github_api.create_deployment_status(deployment["id"], environment_url, github_api.DeploymentStatus.SUCCESS) + if pr: + print(f"Adding label {label} to PR {pr}") + github_api.add_label(pr, f"lock:{label}") + print(f"Setting badge {badge} to {ref} (red)") + set_badge(github_token, org, repo, badge, ref, "red") + except Exception as e: + print(e, file=sys.stderr) + if manage_environment: + print(f"Updating deployment status for {environment_name}") + github_api.create_deployment_status(deployment["id"], environment_url, github_api.DeploymentStatus.ERROR) + fail(pr, 500, f"Deployment to {label} failed for an unknown reason. Please check the logs.") + + +__main__() diff --git a/.github/workflows/deploy-labeled.yml b/.github/workflows/deploy-labeled.yml new file mode 100644 index 000000000000..28a4d259deaf --- /dev/null +++ b/.github/workflows/deploy-labeled.yml @@ -0,0 +1,48 @@ +name: Deploy to Pyris Test + +on: + pull_request: + types: [labeled] + +jobs: + compute-tag: + needs: [ check-build-status ] + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.compute-tag.outputs.result }} + label: ${{ steps.strip-label-prefix.outputs.result }} + steps: + - name: Compute Tag + uses: actions/github-script@v6 + id: compute-tag + with: + result-encoding: string + script: | + if (context.eventName === "pull_request") { + return "pr-" + context.issue.number; + } + if (context.eventName === "release") { + return "latest"; + } + if (context.eventName === "push") { + if (context.ref.startsWith("refs/tags/")) { + return context.ref.slice(10); + } + if (context.ref === "refs/heads/develop") { + return "develop"; + } + } + return "FALSE"; + + - name: Strip label prefix "deploy:" + id: strip-label-prefix + run: echo "::set-output name=result::${{ github.event.label.name:7 }}" + + deploy: + needs: [ compute-tag, pre-deployment ] + uses: ./.github/workflows/deploy.yml + with: + docker-tag: ${{ needs.compute-tag.outputs.tag }} + ref: ${{ github.event.pull_request.head.ref }} + test-server: ${{ needs.compute-tag.outputs.label }} + secrets: inherit diff --git a/.github/workflows/deploy-manual.yml b/.github/workflows/deploy-manual.yml new file mode 100644 index 000000000000..be3457728cff --- /dev/null +++ b/.github/workflows/deploy-manual.yml @@ -0,0 +1,27 @@ +name: Deploy Artemis + +on: + workflow_dispatch: + inputs: + docker-tag: + description: 'Docker tag to deploy (e.g. 1.0.0 or latest, default: latest)' + required: true + default: 'latest' + branch-name: + description: 'Branch name to deploy (default: develop)' + required: true + default: 'develop' + test-server: + type: environment + description: 'Test server to deploy to' + required: true + + +jobs: + deploy: + uses: ./.github/workflows/deploy.yml + with: + docker-tag: ${{ github.event.inputs.docker-tag }} + branch-name: ${{ github.event.inputs.branch-name }} + test-server: ${{ github.event.inputs.test-server }} + secrets: inherit diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000000..cec3e21ed7be --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,66 @@ +name: Deploy Artemis + +on: + workflow_call: + inputs: + docker-tag: + required: true + type: string + ref: + required: true + type: string + pr-number: + required: false + type: number + test-server: + required: true + type: string + secrets: + GITHUB_TOKEN: + required: true + GITHUB_ORG_TOKEN: + required: false + DEPLOYMENT_GATEWAY_SSH_KEY: + required: true + +concurrency: test-servers-deployment + +jobs: + deploy: + needs: [ process-matrix ] + runs-on: ubuntu-latest + + environment: + name: ${{ inputs.test-server }}.artemis.cit.tum.de + url: https://${{ inputs.test-server }}.artemis.cit.tum.de + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run python script to configure SSH Key + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ORG_TOKEN: ${{ secrets.GITHUB_ORG_TOKEN }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_USER: ${{ github.actor.login }} + ORG: "ls1intum" + REPO: "Artemis" + REF: ${{ inputs.ref }} + PR: ${{ inputs.pr-number }} + LABEL: ${{ inputs.test-server }} + TAG: ${{ inputs.docker-tag }} + DEPLOYMENT_USER: ${{ env.DEPLOYMENT_USER }} + DEPLOYMENT_HOSTS: ${{ env.DEPLOYMENT_HOSTS }} + DEPLOYMENT_FOLDER: ${{ env.DEPLOYMENT_FOLDER }} + DEPLOYMENT_HOST_PUBLIC_KEYS: ${{ env.DEPLOYMENT_HOST_PUBLIC_KEYS }} + DEPLOYMENT_SSH_KEY: "${{ secrets.DEPLOYMENT_SSH_KEY }}" + GATEWAY_USER: "jump" + GATEWAY_HOST: "gateway.artemis.in.tum.de:2010" + GATEWAY_HOST_PUBLIC_KEYS: "[gateway.artemis.in.tum.de]:2010 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKtTLiKRILjKZ+Qg4ReWKsG7mLDXkzHfeY5nalSQUNQ4" + GATEWAY_SSH_KEY: "${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}" + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: python3 .ci/deployment/main.py + + +