Avoid extra file copying in setup of test environment (#40)

.pylintrc

@@ -302,7 +302,7 @@ max-bool-expr=5
 max-branches=14
 
 # Maximum number of locals for function / method body.
-max-locals=17
+max-locals=18
 
 # Maximum number of parents for a class (see R0901).
 max-parents=7

ch_backup/ch_backup.py

@@ -117,7 +117,7 @@ def backup(
 
         If force is True, backup.min_interval config option is ignored.
         """
-        # pylint: disable=too-many-locals,too-many-branches
+        # pylint: disable=too-many-branches
         logging.info(f"Backup sources: {sources}")
         assert not (db_names and tables)
 
@@ -206,7 +206,7 @@ def backup(
 
         return self._context.backup_meta.name, None
 
-    # pylint: disable=too-many-arguments,too-many-locals,duplicate-code
+    # pylint: disable=too-many-arguments,duplicate-code
     def restore(
         self,
         sources: BackupSources,
@@ -304,7 +304,7 @@ def restore(
                 keep_going=keep_going,
             )
 
-    # pylint: disable=too-many-locals,too-many-nested-blocks,too-many-branches
+    # pylint: disable=too-many-nested-blocks,too-many-branches
     def fix_s3_oplog(
         self,
         source_cluster_id: str = None,

tests/integration/env_control.py

@@ -14,9 +14,8 @@
 SESSION_STATE_CONF = ".session_conf.sav"
 STAGES = {
     "create": [
-        docker.prep_images,
         compose.create_config,
-        templates.render_configs,
+        templates.render_docker_configs,
         compose.build_images,
     ],
     "start": [
@@ -26,9 +25,8 @@
         minio.create_s3_buckets,
     ],
     "update": [
-        docker.prep_images,
         compose.create_config,
-        templates.render_configs,
+        templates.render_docker_configs,
     ],
     "restart": [
         compose.shutdown_containers,

tests/integration/modules/docker.py

@@ -7,7 +7,6 @@
 import random
 import re
 import tarfile
-from distutils import dir_util  # pylint: disable=deprecated-module
 from typing import List, Sequence, Tuple
 from urllib.parse import urlparse
 
@@ -103,21 +102,6 @@ def get_file_size(container: Container, path: str) -> int:
     return int(output.decode())
 
 
-@utils.env_stage("create", fail=True)
-def prep_images(context: ContextT) -> None:
-    """
-    Prepare images.
-    """
-    images_dir = context.conf["images_dir"]
-    staging_dir = context.conf["staging_dir"]
-    for name, conf in context.conf["services"].items():
-        for i in range(1, conf.get("docker_instances", 1) + 1):
-            dir_util.copy_tree(
-                f"{images_dir}/{name}",
-                f"{staging_dir}/images/{name}{i:02d}",
-            )
-
-
 @utils.env_stage("create", fail=True)
 def create_network(context: ContextT) -> None:
     """

tests/integration/modules/templates.py

@@ -2,6 +2,7 @@
 Module responsible for template rendering.
 """
 import os
+import shutil
 
 from jinja2 import BaseLoader, Environment, FileSystemLoader, StrictUndefined
 
@@ -10,29 +11,7 @@
 from .typing import ContextT
 from .utils import context_to_dict, env_stage, version_ge, version_lt
 
-TEMP_FILE_EXT = "temp~"
-IGNORED_EXT_LIST = [TEMP_FILE_EXT, "gpg"]
-
-
-@env_stage("create", fail=True)
-def render_configs(context: ContextT) -> None:
-    """
-    Render each template in the subtree.
-    Each template is rendered in-place. As the framework operates in
-    staging dir, this is easily reset by `make clean`, or `rm -fr staging`.
-    """
-    staging_dir = context.conf["staging_dir"]
-    for service, conf in context.conf["services"].items():
-        for i in range(1, conf.get("docker_instances", 1) + 1):
-            instance_dir = f"{staging_dir}/images/{service}{i:02d}"
-            context.instance_id = f"{i:02d}"
-            context.instance_name = f"{service}{i:02d}"
-            for root, _, files in os.walk(instance_dir):
-                for filename in files:
-                    if not _is_ignored(filename):
-                        _render_file(context, root, filename)
-    context.instance_name = None
-    context.instance_id = None
+IGNORED_EXT_LIST = ["gpg"]
 
 
 def render_template(context: ContextT, text: str) -> str:
@@ -43,27 +22,69 @@ def render_template(context: ContextT, text: str) -> str:
     return template.render(context_to_dict(context))
 
 
-def _is_ignored(filename):
+@env_stage("create", fail=True)
+def render_docker_configs(context: ContextT) -> None:
+    """
+    Render templated Docker configs.
+    """
+    images_dir = context.conf["images_dir"]
+    staging_dir = context.conf["staging_dir"]
+    for service_name, conf in context.conf["services"].items():
+        service_dir = os.path.join(images_dir, service_name)
+        for i in range(1, conf.get("docker_instances", 1) + 1):
+            instance_id = f"{i:02d}"
+            instance_name = f"{service_name}{instance_id}"
+            instance_dir = os.path.join(staging_dir, "images", instance_name)
+            os.makedirs(instance_dir, exist_ok=True)
+            for dirpath, dirnames, filenames in os.walk(service_dir):
+                target_dir = os.path.join(
+                    instance_dir, os.path.relpath(dirpath, start=service_dir)
+                )
+                for dirname in dirnames:
+                    os.makedirs(os.path.join(target_dir, dirname), exist_ok=True)
+
+                for filename in filenames:
+                    source_path = os.path.join(dirpath, filename)
+                    target_path = os.path.join(target_dir, filename)
+                    if _is_template(source_path):
+                        _render_file(
+                            context=context,
+                            source_path=source_path,
+                            target_path=target_path,
+                            instance_id=instance_id,
+                            instance_name=instance_name,
+                        )
+                    else:
+                        shutil.copy(source_path, target_path)
+
+
+def _is_template(source_path):
     for ignored_ext in IGNORED_EXT_LIST:
-        if filename.endswith(ignored_ext):
-            return True
+        if source_path.endswith(ignored_ext):
+            return False
+
+    return True
 
-    return False
 
+def _render_file(
+    context: ContextT,
+    source_path: str,
+    target_path: str,
+    instance_id: str,
+    instance_name: str,
+) -> None:
+    environment = _environment(context, FileSystemLoader("."))
 
-def _render_file(context: ContextT, directory: str, basename: str) -> None:
-    path = os.path.join(directory, basename)
-    temp_file_path = f"{path}.{TEMP_FILE_EXT}"
-    loader = FileSystemLoader(directory)
-    environment = _environment(context, loader)
     jinja_context = context_to_dict(context)
+    jinja_context["instance_id"] = instance_id
+    jinja_context["instance_name"] = instance_name
+
     try:
-        with open(temp_file_path, "w", encoding="utf-8") as temp_file:
-            template = environment.get_template(basename)
-            temp_file.write(template.render(jinja_context))
+        with open(target_path, "w", encoding="utf-8") as file:
+            template = environment.get_template(source_path)
+            file.write(template.render(jinja_context))
     except Exception as e:
-        raise RuntimeError(f"Failed to render {path}") from e
-    os.rename(temp_file_path, path)
+        raise RuntimeError(f"Failed to render {target_path}") from e
 
 
 def _environment(context: ContextT, loader: BaseLoader = None) -> Environment: