Resolve merge conflict

.github/workflows/workflow.yml

@@ -24,8 +24,8 @@ jobs:
       run: make isort
     - name: "lint: black"
       run: make black
-    - name: "lint: flake8"
-      run: make flake8
+    - name: "lint: ruff"
+      run: make ruff
     - name: "lint: pylint"
       run: make pylint
     - name: "lint: mypy"
@@ -48,7 +48,7 @@ jobs:
         - "22.3.20.29"
         - "22.8.20.11"
         - "23.3.8.21"
-        - "23.7.1.2470"
+        - "latest"
     runs-on: ubuntu-${{ matrix.target.ubuntu }}
     steps:
     - uses: actions/checkout@v3

.ruff.toml

@@ -0,0 +1,9 @@
+select = ["E", "F", "W", "A"]
+
+ignore = [
+    "A003",  # "Class attribute is shadowing a Python builtin"
+    "E501",  # "Line too long"
+]
+
+[per-file-ignores]
+"__init__.py" = ["F401"]  # "Unused import"

Makefile

@@ -2,7 +2,7 @@ export PYTHON?=python3
 export PYTHONIOENCODING?=utf8
 export NO_VENV?=
 export COMPOSE_HTTP_TIMEOUT?=300
-export CLICKHOUSE_VERSION?=23.3.7.5
+export CLICKHOUSE_VERSION?=latest
 
 ifndef NO_VENV
   export PATH := venv/bin:${PATH}
@@ -27,7 +27,7 @@ build: install-deps ch_backup/version.txt
 all: build lint test-unit test-integration
 
 .PHONY: lint
-lint: install-deps isort black flake8 pylint mypy bandit
+lint: install-deps isort black ruff pylint mypy bandit
 
 .PHONY: isort
 isort: install-deps
@@ -37,9 +37,9 @@ isort: install-deps
 black: install-deps
 	${TEST_ENV} black --check --diff .
 
-.PHONY: flake8
-flake8: install-deps
-	${TEST_ENV} flake8 ch_backup tests
+.PHONY: ruff
+ruff: install-deps
+	${TEST_ENV} ruff check ch_backup tests
 
 .PHONY: pylint
 pylint: install-deps
@@ -157,6 +157,7 @@ install-deps: .install-deps
 
 .install-deps: requirements.txt requirements-dev.txt
 	if [ -z "${NO_VENV}" ]; then ${PYTHON} -m venv venv; fi
+	${TEST_ENV} pip install --upgrade pip
 	${TEST_ENV} pip install --no-cache-dir --disable-pip-version-check -r requirements.txt -r requirements-dev.txt
 	touch .install-deps
 
@@ -166,12 +167,12 @@ help:
 	@echo "Targets:"
 	@echo "  build (default)            Build project. It installs dependencies and generates version.txt."
 	@echo "  all                        Alias for \"build lint test-unit test-integration\"."
-	@echo "  lint                       Run all linter tools. Alias for \"isort black flake8 pylint mypy bandit\"."
+	@echo "  lint                       Run all linter tools. Alias for \"isort black ruff pylint mypy bandit\"."
 	@echo "  test-unit                  Run unit tests."
 	@echo "  test-integration           Run integration tests."
 	@echo "  isort                      Perform isort checks."
 	@echo "  black                      Perform black checks."
-	@echo "  flake8                     Perform flake8 checks."
+	@echo "  ruff                       Perform ruff checks."
 	@echo "  pylint                     Perform pylint checks."
 	@echo "  mypy                       Perform mypy checks.."
 	@echo "  bandit                     Perform bandit checks."

README.md

@@ -44,7 +44,7 @@ Commands:
 
 ### Regression
 
-The regression test suite contains run of static code analysis tools (isort, black, flake8, pylint, mypy, bandit),
+The regression test suite contains run of static code analysis tools (isort, black, ruff, pylint, mypy, bandit),
 unit tests and integration tests.
 
 The tests can be run by issuing the command:

ch_backup/ch_backup.py

@@ -31,7 +31,7 @@
 from ch_backup.logic.database import DatabaseBackup
 from ch_backup.logic.table import TableBackup
 from ch_backup.logic.udf import UDFBackup
-from ch_backup.util import now, utcnow
+from ch_backup.util import cached_property, now, utcnow
 from ch_backup.version import get_version
 
 
@@ -44,40 +44,45 @@ class ClickhouseBackup:
     # pylint: disable=too-many-instance-attributes
 
     def __init__(self, config: Config) -> None:
-        self._context: BackupContext = self._load_context(config)
+        self._config = config
         self._access_backup_manager = AccessBackup()
         self._database_backup_manager = DatabaseBackup()
         self._table_backup_manager = TableBackup()
         self._udf_backup_manager = UDFBackup()
 
-    @staticmethod
-    def _load_context(config: Config) -> BackupContext:
-        ctx = BackupContext(config)
-
-        ctx.ch_ctl_conf = config["clickhouse"]
-        ctx.main_conf = config["main"]
-        ctx.ch_ctl = ClickhouseCTL(ctx.ch_ctl_conf, ctx.main_conf)
-        ctx.backup_layout = BackupLayout(config)
-        ctx.config = config["backup"]
-        ctx.zk_config = config.get("zookeeper")
-        ctx.restore_context = RestoreContext(ctx.config)
-        ctx.ch_config = ClickhouseConfig(config)
-
-        return ctx
-
     @property
     def config(self) -> Config:
         """
         Returns current config.
         """
-        return self._context.config_root
+        return self._config
+
+    @cached_property
+    def _context(self) -> BackupContext:
+        """
+        Create and configure BackupContext
+        """
+
+        ctx = BackupContext(self._config)
+        ctx.ch_ctl_conf = self._config["clickhouse"]
+        ctx.main_conf = self._config["main"]
+
+        ctx.ch_ctl = ClickhouseCTL(ctx.ch_ctl_conf, ctx.main_conf)
+        ctx.backup_layout = BackupLayout(self._config)
+
+        ctx.config = self._config["backup"]
+        ctx.zk_config = self._config.get("zookeeper")
+        ctx.restore_context = RestoreContext(ctx.config)
+        ctx.ch_config = ClickhouseConfig(self._config)
+        return ctx
 
     def reload_config(self, config: Config) -> None:
         """
         Completely reloads the config.
         """
         logging.info("Reloading config.")
-        self._context = self._load_context(config)
+        del self._context
+        self._config = config
         logging.info("Config reloaded.")
 
     def get(self, backup_name: str) -> BackupMetadata:

ch_backup/clickhouse/control.py

@@ -4,6 +4,7 @@
 
 import os
 import shutil
+from contextlib import contextmanager, suppress
 from hashlib import md5
 from pathlib import Path
 from tarfile import BLOCKSIZE  # type: ignore
@@ -510,11 +511,12 @@ def drop_table_if_exists(self, table: Table) -> None:
         """
         Drop table. If the specified table doesn't exist, do nothing.
         """
-        self._ch_client.query(
-            DROP_TABLE_IF_EXISTS_SQL.format(
-                db_name=escape(table.database), table_name=escape(table.name)
+        with self._force_drop_table():
+            self._ch_client.query(
+                DROP_TABLE_IF_EXISTS_SQL.format(
+                    db_name=escape(table.database), table_name=escape(table.name)
+                )
             )
-        )
 
     def drop_dictionary_if_exists(self, table: Table) -> None:
         """
@@ -530,9 +532,10 @@ def drop_database_if_exists(self, db_name: str) -> None:
         """
         Drop database. If the specified database doesn't exist, do nothing.
         """
-        self._ch_client.query(
-            DROP_DATABASE_IF_EXISTS_SQL.format(db_name=escape(db_name))
-        )
+        with self._force_drop_table():
+            self._ch_client.query(
+                DROP_DATABASE_IF_EXISTS_SQL.format(db_name=escape(db_name))
+            )
 
     def drop_udf(self, udf_name: str) -> None:
         """
@@ -779,6 +782,25 @@ def reload_config(self):
         """
         self._ch_client.query(RELOAD_CONFIG_SQL, timeout=self._timeout)
 
+    @staticmethod
+    @contextmanager
+    def _force_drop_table():
+        """
+        Set and clear on exit force_drop_table flag.
+
+        If it has been set before then don't touch it.
+        This flag allows to overcome TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT ClickHouse error.
+        """
+        flag_path = Path("/var/lib/clickhouse/flags/force_drop_table")
+
+        flag_path.touch()
+        try:
+            flag_path.chmod(0o666)
+            yield
+        finally:
+            with suppress(FileNotFoundError):
+                flag_path.unlink()
+
 
 def _get_part_checksum(part_path: str) -> str:
     with open(os.path.join(part_path, "checksums.txt"), "rb") as f:

ch_backup/type_hints/boto3/s3.py

@@ -4,8 +4,8 @@
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3 import S3Client
-    from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef
+    from mypy_boto3_s3 import S3Client  # noqa: F401
+    from mypy_boto3_s3.type_defs import ObjectIdentifierTypeDef  # noqa: F401
 else:
     # TODO: Use module level __getattr_() as fallback (PEP 562) in Python 3.7+
     S3Client = Any

ch_backup/util.py

@@ -401,3 +401,21 @@ def grant(self, tokens=1):
             return True
 
         return False
+
+
+# pylint: disable=invalid-name
+class cached_property:
+    """
+    Analogue for functools.cached_property.
+
+    We could use cached_property from functools when the supported version of python would be >= 3.8.
+    """
+
+    def __init__(self, func):
+        self.func = func
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+        value = obj.__dict__[self.func.__name__] = self.func(obj)
+        return value

images/clickhouse/Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:bionic
 ENV TZ=Europe/Moscow
 ENV CH_TMP_DIR /var/tmp/ch-backup
 
-ARG CLICKHOUSE_VERSION
+ARG CLICKHOUSE_VERSION=latest
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
     apt-get update -qq && \
@@ -15,7 +15,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone &
         locales \
         python3-pip \
         openssh-server \
-        supervisor && \
+        supervisor \
+        less && \
     pip3 install --upgrade pip && \
     pip3 install kazoo && \
     echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \
@@ -39,10 +40,17 @@ RUN cd ${CH_TMP_DIR} && \
     mkdir -p /etc/apt/sources.list.d && \
     echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update -qq && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        clickhouse-common-static=$CLICKHOUSE_VERSION \
-        clickhouse-server=$CLICKHOUSE_VERSION \
-        clickhouse-client=$CLICKHOUSE_VERSION && \
+    if [ "${CLICKHOUSE_VERSION}" = "latest" ]; then \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            clickhouse-server \
+            clickhouse-client \
+            clickhouse-common-static; \
+    else \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            clickhouse-server=${CLICKHOUSE_VERSION} \
+            clickhouse-client=${CLICKHOUSE_VERSION} \
+            clickhouse-common-static=${CLICKHOUSE_VERSION}; \
+    fi && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

images/clickhousenozk/Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:bionic
 ENV TZ=Europe/Moscow
 ENV CH_TMP_DIR /var/tmp/ch-backup
 
-ARG CLICKHOUSE_VERSION
+ARG CLICKHOUSE_VERSION=latest
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
     apt-get update -qq && \
@@ -15,7 +15,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone &
         locales \
         python3-pip \
         openssh-server \
-        supervisor && \
+        supervisor \
+        less && \
     pip3 install --upgrade pip && \
     pip3 install kazoo && \
     echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && \
@@ -39,10 +40,17 @@ RUN cd ${CH_TMP_DIR} && \
     mkdir -p /etc/apt/sources.list.d && \
     echo "deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb stable main" | tee /etc/apt/sources.list.d/clickhouse.list && \
     apt-get update -qq && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y \
-        clickhouse-common-static=$CLICKHOUSE_VERSION \
-        clickhouse-server=$CLICKHOUSE_VERSION \
-        clickhouse-client=$CLICKHOUSE_VERSION && \
+    if [ "${CLICKHOUSE_VERSION}" = "latest" ]; then \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            clickhouse-server \
+            clickhouse-client \
+            clickhouse-common-static; \
+    else \
+        DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            clickhouse-server=${CLICKHOUSE_VERSION} \
+            clickhouse-client=${CLICKHOUSE_VERSION} \
+            clickhouse-common-static=${CLICKHOUSE_VERSION}; \
+    fi && \
     rm -rf /var/lib/apt/lists/* /var/cache/debconf && \
     apt-get clean
 

images/clickhousenozk/config/supervisor/conf.d/clickhouse.conf

@@ -2,7 +2,6 @@
 command=/usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml
 process_name=%(program_name)s
 autostart=true
-autorestart=true
 stopsignal=TERM
 user=clickhouse
 stdout_logfile=/dev/stderr

requirements-dev.txt

@@ -1,8 +1,6 @@
 isort
 black
-flake8
-flake8-string-format
-flake8-print
+ruff; python_version >="3.7"
 pep8
 pylint
 mypy==0.910

tests/integration/ch_backup.featureset

@@ -20,3 +20,4 @@ features/ssl_support.feature
 features/table_engines.feature
 features/udf_support.feature
 features/traffic_limiting.feature
+features/backup_misc_commands.feature

tests/integration/features/backup_misc_commands.feature

@@ -0,0 +1,12 @@
+
+Feature: Commands with shutted down ch server.
+
+  Background:
+    Given default configuration
+    And a working s3
+    And a working clickhouse on clickhousenozk02
+
+
+  Scenario: Version command without clickhouse server
+    When we stop clickhouse at clickhousenozk02
+    Then we got a valid ch-backup version on clickhousenozk02