From ea6cde03bd7d2389801a30da0c425e252827686d Mon Sep 17 00:00:00 2001 From: Slaytanic Date: Mon, 7 Sep 2020 11:15:59 +0800 Subject: [PATCH 1/5] Determine if nbresuse running in docker Get resources and usages in docker, and return the correct values i think --- nbresuse/api.py | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/nbresuse/api.py b/nbresuse/api.py index 8f2ba5f..c65b63b 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -28,12 +28,25 @@ async def get(self): all_processes = [cur_process] + cur_process.children(recursive=True) # Get memory information - rss = sum([p.memory_info().rss for p in all_processes]) - - if callable(config.mem_limit): - mem_limit = config.mem_limit(rss=rss) - else: # mem_limit is an Int - mem_limit = config.mem_limit + dockerized = self.__if_dockerized() + if not dockerized: + # if running on physical server + rss = sum([p.memory_info().rss for p in all_processes]) + # unused rss, use system avail memory instead + else: + # if running in docker container or pod + with open('/sys/fs/cgroup/memory/memory.usage_in_bytes') as usage: + rss = int(usage.read().strip()) + + if not dockerized: + if callable(config.mem_limit): + mem_limit = config.mem_limit(rss=rss) + else: # mem_limit is an Int + mem_limit = config.mem_limit + else: + # for docker use only + with open('/sys/fs/cgroup/memory/memory.limit_in_bytes') as limit: + mem_limit = int(limit.read().strip()) limits = {"memory": {"rss": mem_limit}} if config.mem_limit and config.mem_warning_threshold != 0: @@ -45,8 +58,14 @@ async def get(self): # Optionally get CPU information if config.track_cpu_percent: - cpu_count = psutil.cpu_count() - cpu_percent = await self._get_cpu_percent(all_processes) + if not dockerized: + # original nbresuse + cpu_count = psutil.cpu_count() + cpu_percent = await self._get_cpu_percent(all_processes) + else: + # for docker use only + cpu_count = int(open('/sys/fs/cgroup/cpu/cpu.cfs_quota_us').read().strip()) / 100000 + cpu_percent = await self._get_cpu_percent(all_processes) / cpu_count if config.cpu_limit != 0: limits["cpu"] = {"cpu": config.cpu_limit} @@ -70,3 +89,12 @@ def get_cpu_percent(p): return 0 return sum([get_cpu_percent(p) for p in all_processes]) + + @staticmethod + def __if_dockerized(): + if os.path.exists('/.dockerenv'): + # running in docker or pod + return True + else: + # running on physical server + return False From 8ac57abbc9c9ea021079a4806413df699799b114 Mon Sep 17 00:00:00 2001 From: Slaytanic Date: Mon, 7 Sep 2020 11:23:58 +0800 Subject: [PATCH 2/5] forgot import os module --- nbresuse/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nbresuse/api.py b/nbresuse/api.py index c65b63b..b53b3fc 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -5,6 +5,7 @@ from notebook.base.handlers import IPythonHandler from tornado import web from tornado.concurrent import run_on_executor +import os try: # Traitlets >= 4.3.3 From 72a3ba1cbb899a7d84348c3b8a9807ec474a6914 Mon Sep 17 00:00:00 2001 From: Slaytanic Date: Tue, 8 Sep 2020 15:13:10 +0800 Subject: [PATCH 3/5] Update api.py --- nbresuse/api.py | 68 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/nbresuse/api.py b/nbresuse/api.py index b53b3fc..464f035 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -29,30 +29,27 @@ async def get(self): all_processes = [cur_process] + cur_process.children(recursive=True) # Get memory information + # If running in a docker container or pod dockerized = self.__if_dockerized() if not dockerized: # if running on physical server rss = sum([p.memory_info().rss for p in all_processes]) - # unused rss, use system avail memory instead - else: - # if running in docker container or pod - with open('/sys/fs/cgroup/memory/memory.usage_in_bytes') as usage: - rss = int(usage.read().strip()) - - if not dockerized: if callable(config.mem_limit): mem_limit = config.mem_limit(rss=rss) else: # mem_limit is an Int mem_limit = config.mem_limit + # unused rss, use system avail memory instead else: - # for docker use only - with open('/sys/fs/cgroup/memory/memory.limit_in_bytes') as limit: - mem_limit = int(limit.read().strip()) + # if running in docker container or pod + mem_limit = self.__get_docker_physical_memory() + rss = self.__get_docker_memory_usage() + real_rss = self.__get_docker_real_rss_memory() + rss = rss - real_rss limits = {"memory": {"rss": mem_limit}} - if config.mem_limit and config.mem_warning_threshold != 0: + if config.mem_limit: limits["memory"]["warn"] = (mem_limit - rss) < ( - mem_limit * config.mem_warning_threshold + mem_limit * config.mem_warning_threshold ) metrics = {"rss": rss, "limits": limits} @@ -72,7 +69,7 @@ async def get(self): limits["cpu"] = {"cpu": config.cpu_limit} if config.cpu_warning_threshold != 0: limits["cpu"]["warn"] = (config.cpu_limit - cpu_percent) < ( - config.cpu_limit * config.cpu_warning_threshold + config.cpu_limit * config.cpu_warning_threshold ) metrics.update(cpu_percent=cpu_percent, cpu_count=cpu_count) @@ -88,9 +85,49 @@ def get_cpu_percent(p): # about dead processes having no CPU usage except: return 0 - return sum([get_cpu_percent(p) for p in all_processes]) - + + @staticmethod + def __get_docker_physical_memory(): + with open('/sys/fs/cgroup/memory/memory.limit_in_bytes') as limit: + mem_limit = int(limit.read().strip()) + return mem_limit + + @staticmethod + def __get_docker_memory_usage(): + with open('/sys/fs/cgroup/memory/memory.usage_in_bytes') as usage: + all_used = int(usage.read().strip()) + return all_used + + @staticmethod + def __get_docker_real_rss_memory(): + real_rss = 0 + with open('/sys/fs/cgroup/memory/memory.stat') as stats: + for line in stats.readlines(): + stat = line.split() + if stat[0] in ['rss', 'inactive_file', 'active_file']: + real_rss = real_rss + int(stat[1]) + return real_rss + + def _check_mem_used(self): + memory_limit = 20 * 1024 * 1024 * 1024 + sysmem = self.__get_docker_physical_memory() + rss = self.__get_docker_memory_usage() + + real_rss = self.__get_docker_real_rss_memory() + + avail_mem = rss - real_rss + if sysmem < memory_limit: + if round(avail_mem / sysmem * 100) < 20: + return True + else: + return False + else: + if avail_mem < 5 * 1024 * 1024 * 1024: + return True + else: + return False + @staticmethod def __if_dockerized(): if os.path.exists('/.dockerenv'): @@ -99,3 +136,4 @@ def __if_dockerized(): else: # running on physical server return False + From d4ec720c9cba49030116f46f54aeb4c5073b48dc Mon Sep 17 00:00:00 2001 From: Slaytanic Date: Tue, 8 Sep 2020 15:14:05 +0800 Subject: [PATCH 4/5] Update api.py --- nbresuse/api.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/nbresuse/api.py b/nbresuse/api.py index 464f035..596f7d7 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -109,25 +109,6 @@ def __get_docker_real_rss_memory(): real_rss = real_rss + int(stat[1]) return real_rss - def _check_mem_used(self): - memory_limit = 20 * 1024 * 1024 * 1024 - sysmem = self.__get_docker_physical_memory() - rss = self.__get_docker_memory_usage() - - real_rss = self.__get_docker_real_rss_memory() - - avail_mem = rss - real_rss - if sysmem < memory_limit: - if round(avail_mem / sysmem * 100) < 20: - return True - else: - return False - else: - if avail_mem < 5 * 1024 * 1024 * 1024: - return True - else: - return False - @staticmethod def __if_dockerized(): if os.path.exists('/.dockerenv'): From 8aec768870bcf20be87e5b9287a346e57c853e5b Mon Sep 17 00:00:00 2001 From: Slaytanic Date: Thu, 10 Sep 2020 09:55:46 +0800 Subject: [PATCH 5/5] fix calculate issue --- nbresuse/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nbresuse/api.py b/nbresuse/api.py index 596f7d7..9d3e933 100644 --- a/nbresuse/api.py +++ b/nbresuse/api.py @@ -42,9 +42,8 @@ async def get(self): else: # if running in docker container or pod mem_limit = self.__get_docker_physical_memory() - rss = self.__get_docker_memory_usage() real_rss = self.__get_docker_real_rss_memory() - rss = rss - real_rss + rss = real_rss limits = {"memory": {"rss": mem_limit}} if config.mem_limit: