Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fallback to slurm for TorchDistributedEnv #1706

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fallback to slurm for TorchDistributedEnv #1706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0949301
Default to slurm for TorchDistributedEnv
jrapin Aug 3, 2022
3b32459
fix
jrapin Aug 3, 2022
0a2b451
fix
jrapin Aug 3, 2022
973e7e9
preparenewversion
jrapin Aug 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion submitit/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@
from .slurm.slurm import SlurmExecutor as SlurmExecutor
from .slurm.slurm import SlurmJob as SlurmJob

__version__ = "1.4.5"
__version__ = "1.4.6"
15 changes: 14 additions & 1 deletion submitit/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from .core.utils import CommandFunction as CommandFunction # noqa
from .core.utils import DelayedSubmission as DelayedSubmission # noqa
from .core.utils import environment_variables as environment_variables # noqa
from .slurm.slurm import SlurmJobEnvironment


class Checkpointable:
Expand Down Expand Up @@ -331,14 +332,26 @@ def __init__(self) -> None:
>>> torch.distributed.init_process_group(backend="nccl")
>>> print(f"master: {dist_env.master_addr}:{dist_env.master_port}")
"""
self._job_env = JobEnvironment()
try:
self._job_env = JobEnvironment()
except RuntimeError as e:
if SlurmJobEnvironment._env["job_id"] in os.environ:
# identified a slurm env without submitit, so let's use it
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed, this is a really weird use case and a surprising thing to try to fix: this is basically to make it possible for users (that are not using submitit to launch jobs) to use a helper function from submitit...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how is that a problem? I'm actually happy that we can avoid some inter-dependencies, it gives more freedom

self._job_env = SlurmJobEnvironment()
else:
raise e
self.master_addr = self._job_env.hostnames[0]
self.master_port = self._get_master_port()
self.rank = self._job_env.global_rank
self.world_size = self._job_env.num_tasks
self.local_rank = self._job_env.local_rank
self.local_world_size = self._job_env.num_tasks // self._job_env.num_nodes

def __repr__(self) -> str:
cls = self.__class__.__name__
env = sorted(f"{name}={val}" for name, val in self.__dict__.items() if not name.startswith("_"))
return f"{cls}<{','.join(env)}>"

def _get_master_port(self) -> int:
# MIN_MASTER_PORT, MAX_MASTER_PORT = (1023, 65535)
MIN_MASTER_PORT, MAX_MASTER_PORT = (20000, 60000)
Expand Down
9 changes: 9 additions & 0 deletions submitit/test_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,15 @@ def _get_env() -> tp.Dict[str, str]:
return {x: y for x, y in os.environ.items() if x.startswith(("SLURM_", "SUBMITIT_"))}


def test_torch_distrib_env() -> None:
with pytest.raises(RuntimeError):
env = helpers.TorchDistributedEnvironment()
with utils.environment_variables(SLURM_JOB_ID=12):
env = helpers.TorchDistributedEnvironment()
# port is deduced from job id
assert env.master_port == 58811


def test_clean_env() -> None:
base = _get_env()
with utils.environment_variables(SLURM_BLUBLU=12, SUBMITIT_BLUBLU=12):
Expand Down