[IMP] Configure human-readable memory limits.

odoo/addons/base/tests/__init__.py

@@ -7,6 +7,7 @@
 from . import test_base
 from . import test_basecase
 from . import test_cache
+from . import test_configmanager
 from . import test_db_cursor
 from . import test_expression
 from . import test_float

odoo/addons/base/tests/config/limit_memory.conf

@@ -0,0 +1,3 @@
+[options]
+limit_memory_hard = 3GiB
+limit_memory_soft = 1536MiB

odoo/addons/base/tests/data/limit_memory_old.conf

@@ -0,0 +1,3 @@
+[options]
+limit_memory_hard = 4294967296
+limit_memory_soft = 1073741824

odoo/addons/base/tests/test_configmanager.py

@@ -0,0 +1,51 @@
+import platform
+import unittest
+
+from odoo.modules.module import get_module_resource
+from odoo.tests import BaseCase
+from odoo.tools.config import configmanager
+
+
+class TestConfigManager(BaseCase):
+    def test_defaults(self):
+        config = configmanager()
+        self.assertEqual(config['limit_memory_hard'], 2684354560)
+        self.assertEqual(config['limit_memory_soft'], 2147483648)
+
+    def test_limit_memory_old(self):
+        config = configmanager(fname=get_module_resource('base', 'tests', 'data', 'limit_memory_old.conf'))
+        self.assertEqual(config['limit_memory_hard'], 4294967296)
+        self.assertEqual(config['limit_memory_soft'], 1073741824)
+
+    IS_POSIX = platform.system() == 'Linux' and platform.machine() == 'x86_64'
+    @unittest.skipIf(not IS_POSIX, 'this test is POSIX only')
+    def test_04_parse_size(self):
+        config = configmanager(fname=get_module_resource('base', 'tests', 'config', 'limit_memory.conf'))
+        self.assertEqual(config['limit_memory_hard'], 3221225472)
+        self.assertEqual(config['limit_memory_soft'], 1610612736)
+
+        config._parse_config(['--limit-memory-hard', '4GiB', '--limit-memory-soft', '3GiB'])
+        self.assertEqual(config['limit_memory_hard'], 4294967296)
+        self.assertEqual(config['limit_memory_soft'], 3221225472)
+
+        config = configmanager()
+        self.assertEqual(config._parse_size('1024'), 1024)
+        self.assertEqual(config._parse_size('2ki '), 2048)
+        self.assertEqual(config._parse_size(' 4MiB'), 4194304)
+        self.assertEqual(config._parse_size('1 YiB'), 1208925819614629174706176)
+
+        with self.assertRaises(ValueError) as cm:
+            config._parse_size('1.2465')
+        self.assertIn("invalid size", str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            config._parse_size('B')
+        self.assertIn("invalid size", str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            config._parse_size('10kB')
+        self.assertIn("invalid IEC 80000-13 binary prefix", str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            config._parse_size('20fiB')
+        self.assertIn("invalid IEC 80000-13 binary prefix", str(cm.exception))

odoo/tools/config.py

@@ -6,6 +6,7 @@
 import optparse
 import glob
 import os
+import re
 import sys
 import tempfile
 import warnings
@@ -317,14 +318,14 @@ def __init__(self, fname=None):
             group.add_option("--workers", dest="workers", my_default=0,
                              help="Specify the number of workers, 0 disable prefork mode.",
                              type="int")
-            group.add_option("--limit-memory-soft", dest="limit_memory_soft", my_default=2048 * 1024 * 1024,
+            group.add_option("--limit-memory-soft", dest="limit_memory_soft", my_default="2048MiB",
                              help="Maximum allowed virtual memory per worker (in bytes), when reached the worker be "
                              "reset after the current request (default 2048MiB).",
-                             type="int")
-            group.add_option("--limit-memory-hard", dest="limit_memory_hard", my_default=2560 * 1024 * 1024,
+                             action="callback", callback=self._parse_size_callback, nargs=1, type="string")
+            group.add_option("--limit-memory-hard", dest="limit_memory_hard", my_default="2560MiB",
                              help="Maximum allowed virtual memory per worker (in bytes), when reached, any memory "
                              "allocation will fail (default 2560MiB).",
-                             type="int")
+                             action="callback", callback=self._parse_size_callback, nargs=1, type="string")
             group.add_option("--limit-time-cpu", dest="limit_time_cpu", my_default=60,
                              help="Maximum allowed CPU time per request (default 60).",
                              type="int")
@@ -492,8 +493,14 @@ def die(cond, msg):
             if getattr(opt, arg) is not None:
                 self.options[arg] = getattr(opt, arg)
             # ... or keep, but cast, the config file value.
-            elif isinstance(self.options[arg], str) and self.casts[arg].type in optparse.Option.TYPE_CHECKER:
-                self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg])
+            elif isinstance(self.options[arg], str):
+                opt_str = '--' + arg.replace('_', '-')
+                option = self.parser.get_option(opt_str)
+                if option and option.callback:
+                    option.callback(option, opt, self.options[arg], self.parser)
+                    self.options[arg] = getattr(self.parser.values, arg)
+                elif self.casts[arg].type in optparse.Option.TYPE_CHECKER:
+                    self.options[arg] = optparse.Option.TYPE_CHECKER[self.casts[arg].type](self.casts[arg], arg, self.options[arg])
 
         self.options['root_path'] = self._normalize(os.path.join(os.path.dirname(__file__), '..'))
         if not self.options['addons_path'] or self.options['addons_path']=='None':
@@ -546,6 +553,28 @@ def die(cond, msg):
         ]
         return opt
 
+    def _parse_size(self, text):
+        # https://en.wikipedia.org/wiki/Binary_prefix
+        pattern = r"""^\s*(?P<size>\d+) # integer
+                       \s*(?P<prefix>\w{2})?B? # IEC 80000-13 binary prefix
+                       \s*$"""
+        match = re.match(pattern, text, re.VERBOSE)
+        if not match:
+            raise ValueError('invalid size: {size}'.format(size=repr(text)))
+        size = int(match['size'])
+        try:
+            exponent = ('', 'ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi').index(match['prefix'] or '')
+        except ValueError:
+            raise ValueError('invalid IEC 80000-13 binary prefix: {prefix}'.format(prefix=repr(match['prefix'])))
+        return round(size * (1024 ** exponent))
+
+    def _parse_size_callback(self, option, opt, value, parser):
+        try:
+            size = self._parse_size(value)
+        except Exception as e:
+            raise optparse.OptionValueError("option %s: %s" % (option, str(e)))
+        setattr(parser.values, option.dest, size)
+
     def _warn_deprecated_options(self):
         if self.options['osv_memory_age_limit']:
             warnings.warn(