From a9a7af471fef01693af3580a0b212ddedd71f58f Mon Sep 17 00:00:00 2001 From: Alexander Burmak Date: Thu, 24 Aug 2023 08:35:28 +0300 Subject: [PATCH] Avoid extra file copying in setup of test environment (#40) --- .pylintrc | 2 +- ch_backup/ch_backup.py | 6 +- tests/integration/env_control.py | 6 +- tests/integration/modules/docker.py | 16 ----- tests/integration/modules/templates.py | 95 ++++++++++++++++---------- 5 files changed, 64 insertions(+), 61 deletions(-) diff --git a/.pylintrc b/.pylintrc index 4ca0098b..98d95aa7 100644 --- a/.pylintrc +++ b/.pylintrc @@ -302,7 +302,7 @@ max-bool-expr=5 max-branches=14 # Maximum number of locals for function / method body. -max-locals=17 +max-locals=18 # Maximum number of parents for a class (see R0901). max-parents=7 diff --git a/ch_backup/ch_backup.py b/ch_backup/ch_backup.py index 88bc81ac..a5022244 100644 --- a/ch_backup/ch_backup.py +++ b/ch_backup/ch_backup.py @@ -117,7 +117,7 @@ def backup( If force is True, backup.min_interval config option is ignored. """ - # pylint: disable=too-many-locals,too-many-branches + # pylint: disable=too-many-branches logging.info(f"Backup sources: {sources}") assert not (db_names and tables) @@ -206,7 +206,7 @@ def backup( return self._context.backup_meta.name, None - # pylint: disable=too-many-arguments,too-many-locals,duplicate-code + # pylint: disable=too-many-arguments,duplicate-code def restore( self, sources: BackupSources, @@ -304,7 +304,7 @@ def restore( keep_going=keep_going, ) - # pylint: disable=too-many-locals,too-many-nested-blocks,too-many-branches + # pylint: disable=too-many-nested-blocks,too-many-branches def fix_s3_oplog( self, source_cluster_id: str = None, diff --git a/tests/integration/env_control.py b/tests/integration/env_control.py index 14e014e1..06fab7f4 100755 --- a/tests/integration/env_control.py +++ b/tests/integration/env_control.py @@ -14,9 +14,8 @@ SESSION_STATE_CONF = ".session_conf.sav" STAGES = { "create": [ - docker.prep_images, compose.create_config, - templates.render_configs, + templates.render_docker_configs, compose.build_images, ], "start": [ @@ -26,9 +25,8 @@ minio.create_s3_buckets, ], "update": [ - docker.prep_images, compose.create_config, - templates.render_configs, + templates.render_docker_configs, ], "restart": [ compose.shutdown_containers, diff --git a/tests/integration/modules/docker.py b/tests/integration/modules/docker.py index 9e588bfb..93a4d3f9 100644 --- a/tests/integration/modules/docker.py +++ b/tests/integration/modules/docker.py @@ -7,7 +7,6 @@ import random import re import tarfile -from distutils import dir_util # pylint: disable=deprecated-module from typing import List, Sequence, Tuple from urllib.parse import urlparse @@ -103,21 +102,6 @@ def get_file_size(container: Container, path: str) -> int: return int(output.decode()) -@utils.env_stage("create", fail=True) -def prep_images(context: ContextT) -> None: - """ - Prepare images. - """ - images_dir = context.conf["images_dir"] - staging_dir = context.conf["staging_dir"] - for name, conf in context.conf["services"].items(): - for i in range(1, conf.get("docker_instances", 1) + 1): - dir_util.copy_tree( - f"{images_dir}/{name}", - f"{staging_dir}/images/{name}{i:02d}", - ) - - @utils.env_stage("create", fail=True) def create_network(context: ContextT) -> None: """ diff --git a/tests/integration/modules/templates.py b/tests/integration/modules/templates.py index 0facd64b..87a11ccb 100644 --- a/tests/integration/modules/templates.py +++ b/tests/integration/modules/templates.py @@ -2,6 +2,7 @@ Module responsible for template rendering. """ import os +import shutil from jinja2 import BaseLoader, Environment, FileSystemLoader, StrictUndefined @@ -10,29 +11,7 @@ from .typing import ContextT from .utils import context_to_dict, env_stage, version_ge, version_lt -TEMP_FILE_EXT = "temp~" -IGNORED_EXT_LIST = [TEMP_FILE_EXT, "gpg"] - - -@env_stage("create", fail=True) -def render_configs(context: ContextT) -> None: - """ - Render each template in the subtree. - Each template is rendered in-place. As the framework operates in - staging dir, this is easily reset by `make clean`, or `rm -fr staging`. - """ - staging_dir = context.conf["staging_dir"] - for service, conf in context.conf["services"].items(): - for i in range(1, conf.get("docker_instances", 1) + 1): - instance_dir = f"{staging_dir}/images/{service}{i:02d}" - context.instance_id = f"{i:02d}" - context.instance_name = f"{service}{i:02d}" - for root, _, files in os.walk(instance_dir): - for filename in files: - if not _is_ignored(filename): - _render_file(context, root, filename) - context.instance_name = None - context.instance_id = None +IGNORED_EXT_LIST = ["gpg"] def render_template(context: ContextT, text: str) -> str: @@ -43,27 +22,69 @@ def render_template(context: ContextT, text: str) -> str: return template.render(context_to_dict(context)) -def _is_ignored(filename): +@env_stage("create", fail=True) +def render_docker_configs(context: ContextT) -> None: + """ + Render templated Docker configs. + """ + images_dir = context.conf["images_dir"] + staging_dir = context.conf["staging_dir"] + for service_name, conf in context.conf["services"].items(): + service_dir = os.path.join(images_dir, service_name) + for i in range(1, conf.get("docker_instances", 1) + 1): + instance_id = f"{i:02d}" + instance_name = f"{service_name}{instance_id}" + instance_dir = os.path.join(staging_dir, "images", instance_name) + os.makedirs(instance_dir, exist_ok=True) + for dirpath, dirnames, filenames in os.walk(service_dir): + target_dir = os.path.join( + instance_dir, os.path.relpath(dirpath, start=service_dir) + ) + for dirname in dirnames: + os.makedirs(os.path.join(target_dir, dirname), exist_ok=True) + + for filename in filenames: + source_path = os.path.join(dirpath, filename) + target_path = os.path.join(target_dir, filename) + if _is_template(source_path): + _render_file( + context=context, + source_path=source_path, + target_path=target_path, + instance_id=instance_id, + instance_name=instance_name, + ) + else: + shutil.copy(source_path, target_path) + + +def _is_template(source_path): for ignored_ext in IGNORED_EXT_LIST: - if filename.endswith(ignored_ext): - return True + if source_path.endswith(ignored_ext): + return False + + return True - return False +def _render_file( + context: ContextT, + source_path: str, + target_path: str, + instance_id: str, + instance_name: str, +) -> None: + environment = _environment(context, FileSystemLoader(".")) -def _render_file(context: ContextT, directory: str, basename: str) -> None: - path = os.path.join(directory, basename) - temp_file_path = f"{path}.{TEMP_FILE_EXT}" - loader = FileSystemLoader(directory) - environment = _environment(context, loader) jinja_context = context_to_dict(context) + jinja_context["instance_id"] = instance_id + jinja_context["instance_name"] = instance_name + try: - with open(temp_file_path, "w", encoding="utf-8") as temp_file: - template = environment.get_template(basename) - temp_file.write(template.render(jinja_context)) + with open(target_path, "w", encoding="utf-8") as file: + template = environment.get_template(source_path) + file.write(template.render(jinja_context)) except Exception as e: - raise RuntimeError(f"Failed to render {path}") from e - os.rename(temp_file_path, path) + raise RuntimeError(f"Failed to render {target_path}") from e def _environment(context: ContextT, loader: BaseLoader = None) -> Environment: