From b9014ef0335f989b7ab8e2ff5436e1b82dd1348c Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 12:13:17 +0200
Subject: [PATCH 1/9] fix: minor improvements to monitor
---
nipype/interfaces/base/support.py | 19 +++++++++---------
nipype/utils/profiler.py | 32 ++++++++++++++++++-------------
2 files changed, 29 insertions(+), 22 deletions(-)
diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py
index 14c8a55da1..88435b2a32 100644
--- a/nipype/interfaces/base/support.py
+++ b/nipype/interfaces/base/support.py
@@ -19,6 +19,7 @@
from ... import logging, config
from ...utils.misc import is_container, rgetcwd
from ...utils.filemanip import md5, hash_infile
+from ...utils.profiler import ResourceMonitor, ResourceMonitorMock
iflogger = logging.getLogger("nipype.interface")
@@ -33,16 +34,8 @@ class RuntimeContext(AbstractContextManager):
def __init__(self, resource_monitor=False, ignore_exception=False):
"""Initialize the context manager object."""
self._ignore_exc = ignore_exception
- _proc_pid = os.getpid()
- if resource_monitor:
- from ...utils.profiler import ResourceMonitor
- else:
- from ...utils.profiler import ResourceMonitorMock as ResourceMonitor
+ self._resmon = resource_monitor
- self._resmon = ResourceMonitor(
- _proc_pid,
- freq=float(config.get("execution", "resource_monitor_frequency", 1)),
- )
def __call__(self, interface, cwd=None, redirect_x=False):
"""Generate a new runtime object."""
@@ -51,6 +44,14 @@ def __call__(self, interface, cwd=None, redirect_x=False):
if cwd is None:
cwd = _syscwd
+ _proc_pid = os.getpid()
+ _ResourceMonitor = ResourceMonitor if self._resmon else ResourceMonitorMock
+ self._resmon = _ResourceMonitor(
+ _proc_pid,
+ cwd=cwd,
+ freq=float(config.get("execution", "resource_monitor_frequency", 1)),
+ )
+
self._runtime = Bunch(
cwd=str(cwd),
duration=None,
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index 2179b29db6..b6d816a748 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -48,23 +48,27 @@ class ResourceMonitor(threading.Thread):
to a file
"""
- def __init__(self, pid, freq=5, fname=None, python=True):
+ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
# Make sure psutil is imported
import psutil
- if freq < 0.2:
- raise RuntimeError("Frequency (%0.2fs) cannot be lower than 0.2s" % freq)
+ # Leave process initialized and make first sample
+ self._process = psutil.Process(pid)
+ _sample = self._sample(cpu_interval=0.2)
- if fname is None:
- fname = ".proc-%d_time-%s_freq-%0.2f" % (pid, time(), freq)
- self._fname = os.path.abspath(fname)
+ # Continue monitor configuration
+ freq = max(freq, 0.2)
+ fname = fname or f".proc-{pid}"
+ self._fname = os.path.abspath(
+ os.path.join(cwd, fname) if cwd is not None else fname
+ )
self._logfile = open(self._fname, "w")
self._freq = freq
self._python = python
- # Leave process initialized and make first sample
- self._process = psutil.Process(pid)
- self._sample(cpu_interval=0.2)
+ # Dump first sample to file
+ print(",".join(_sample), file=self._logfile)
+ self._logfile.flush()
# Start thread
threading.Thread.__init__(self)
@@ -80,7 +84,8 @@ def stop(self):
if not self._event.is_set():
self._event.set()
self.join()
- self._sample()
+ # Dump last sample to file
+ print(",".join(self._sample()), file=self._logfile)
self._logfile.flush()
self._logfile.close()
@@ -132,15 +137,16 @@ def _sample(self, cpu_interval=None):
except psutil.NoSuchProcess:
pass
- print("%f,%f,%f,%f" % (time(), cpu, rss / _MB, vms / _MB), file=self._logfile)
- self._logfile.flush()
+ return (time(), cpu, rss / _MB, vms / _MB)
def run(self):
"""Core monitoring function, called by start()"""
start_time = time()
wait_til = start_time
while not self._event.is_set():
- self._sample()
+ # Dump sample to file
+ print(",".join(self._sample()), file=self._logfile)
+ self._logfile.flush()
wait_til += self._freq
self._event.wait(max(0, wait_til - time()))
From 1a476b4bab4f62fe53fc56e130f8277935349acc Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 12:16:22 +0200
Subject: [PATCH 2/9] fix: maintain consistent interface of mock monitor
---
nipype/utils/profiler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index b6d816a748..510c05aa49 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -32,7 +32,7 @@ def fname(self):
"""Get/set the internal filename"""
return None
- def __init__(self, pid, freq=5, fname=None, python=True):
+ def __init__(self, pid, freq=5, fname=None, cwd=None):
pass
def start(self):
From ae27346c5ab20c3050a2b73d60d970d567c88e57 Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 12:19:45 +0200
Subject: [PATCH 3/9] fix: remove python argument setting for monitor
---
nipype/utils/profiler.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index 510c05aa49..b59bcab60f 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -64,7 +64,6 @@ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
)
self._logfile = open(self._fname, "w")
self._freq = freq
- self._python = python
# Dump first sample to file
print(",".join(_sample), file=self._logfile)
From 888e7fad9774f7a726d31f7cac3d8c8433f15f6d Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 12:26:30 +0200
Subject: [PATCH 4/9] fix: bad variable name
---
nipype/utils/profiler.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index b59bcab60f..a000e81c92 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -54,7 +54,7 @@ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
# Leave process initialized and make first sample
self._process = psutil.Process(pid)
- _sample = self._sample(cpu_interval=0.2)
+ _first_sample = self._sample(cpu_interval=0.2)
# Continue monitor configuration
freq = max(freq, 0.2)
@@ -66,7 +66,7 @@ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
self._freq = freq
# Dump first sample to file
- print(",".join(_sample), file=self._logfile)
+ print(",".join(_first_sample), file=self._logfile)
self._logfile.flush()
# Start thread
@@ -108,6 +108,7 @@ def stop(self):
return retval
def _sample(self, cpu_interval=None):
+ _time = time()
cpu = 0.0
rss = 0.0
vms = 0.0
@@ -136,7 +137,7 @@ def _sample(self, cpu_interval=None):
except psutil.NoSuchProcess:
pass
- return (time(), cpu, rss / _MB, vms / _MB)
+ return (_time, cpu, rss / _MB, vms / _MB)
def run(self):
"""Core monitoring function, called by start()"""
From 8ae6bc525753f364d57374ca2b9017750bf9df8c Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 14:00:58 +0200
Subject: [PATCH 5/9] fix: cast to strings before printing
---
nipype/utils/profiler.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index a000e81c92..0dd5557116 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -66,7 +66,7 @@ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
self._freq = freq
# Dump first sample to file
- print(",".join(_first_sample), file=self._logfile)
+ print(",".join(f"v" for v in _first_sample), file=self._logfile)
self._logfile.flush()
# Start thread
@@ -84,7 +84,7 @@ def stop(self):
self._event.set()
self.join()
# Dump last sample to file
- print(",".join(self._sample()), file=self._logfile)
+ print(",".join(f"v" for v in self._sample()), file=self._logfile)
self._logfile.flush()
self._logfile.close()
@@ -145,7 +145,7 @@ def run(self):
wait_til = start_time
while not self._event.is_set():
# Dump sample to file
- print(",".join(self._sample()), file=self._logfile)
+ print(",".join(f"v" for v in self._sample())), file=self._logfile)
self._logfile.flush()
wait_til += self._freq
self._event.wait(max(0, wait_til - time()))
From 05baec0c8374fa0177209cdebc12f6873c24a489 Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 14:03:21 +0200
Subject: [PATCH 6/9] fix: extra parenthesis
---
nipype/utils/profiler.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index 0dd5557116..fed1f75561 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -145,7 +145,7 @@ def run(self):
wait_til = start_time
while not self._event.is_set():
# Dump sample to file
- print(",".join(f"v" for v in self._sample())), file=self._logfile)
+ print(",".join(f"v" for v in self._sample()), file=self._logfile)
self._logfile.flush()
wait_til += self._freq
self._event.wait(max(0, wait_til - time()))
From 1fcc082ba808380bcf52ffbf1e8bbf79a60e8b03 Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 16:03:37 +0200
Subject: [PATCH 7/9] fix: bad string substitution
---
nipype/utils/profiler.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index fed1f75561..324c6701ea 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -66,7 +66,7 @@ def __init__(self, pid, freq=0.2, fname=None, cwd=None):
self._freq = freq
# Dump first sample to file
- print(",".join(f"v" for v in _first_sample), file=self._logfile)
+ print(",".join(f"{v}" for v in _first_sample), file=self._logfile)
self._logfile.flush()
# Start thread
@@ -84,7 +84,7 @@ def stop(self):
self._event.set()
self.join()
# Dump last sample to file
- print(",".join(f"v" for v in self._sample()), file=self._logfile)
+ print(",".join(f"{v}" for v in self._sample()), file=self._logfile)
self._logfile.flush()
self._logfile.close()
@@ -145,7 +145,7 @@ def run(self):
wait_til = start_time
while not self._event.is_set():
# Dump sample to file
- print(",".join(f"v" for v in self._sample()), file=self._logfile)
+ print(",".join(f"{v}" for v in self._sample()), file=self._logfile)
self._logfile.flush()
wait_til += self._freq
self._event.wait(max(0, wait_til - time()))
From 064bd4b0057608b1640e0572eb968c2b5132abd9 Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 16:22:53 +0200
Subject: [PATCH 8/9] fix: do not use mock
---
nipype/interfaces/base/support.py | 26 ++++++++++++++------------
nipype/utils/profiler.py | 18 ------------------
2 files changed, 14 insertions(+), 30 deletions(-)
diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py
index 88435b2a32..85e6869bac 100644
--- a/nipype/interfaces/base/support.py
+++ b/nipype/interfaces/base/support.py
@@ -19,7 +19,7 @@
from ... import logging, config
from ...utils.misc import is_container, rgetcwd
from ...utils.filemanip import md5, hash_infile
-from ...utils.profiler import ResourceMonitor, ResourceMonitorMock
+from ...utils.profiler import ResourceMonitor
iflogger = logging.getLogger("nipype.interface")
@@ -44,13 +44,13 @@ def __call__(self, interface, cwd=None, redirect_x=False):
if cwd is None:
cwd = _syscwd
- _proc_pid = os.getpid()
- _ResourceMonitor = ResourceMonitor if self._resmon else ResourceMonitorMock
- self._resmon = _ResourceMonitor(
- _proc_pid,
- cwd=cwd,
- freq=float(config.get("execution", "resource_monitor_frequency", 1)),
- )
+ if self._resmon:
+ _proc_pid = os.getpid()
+ self._resmon = ResourceMonitor(
+ _proc_pid,
+ cwd=cwd,
+ freq=float(config.get("execution", "resource_monitor_frequency", 1)),
+ )
self._runtime = Bunch(
cwd=str(cwd),
@@ -62,7 +62,7 @@ def __call__(self, interface, cwd=None, redirect_x=False):
platform=platform.platform(),
prevcwd=str(_syscwd),
redirect_x=redirect_x,
- resmon=self._resmon.fname or "off",
+ resmon=getattr(self._resmon, "fname", "off"),
returncode=None,
startTime=None,
version=interface.version,
@@ -75,7 +75,8 @@ def __enter__(self):
self._runtime.environ["DISPLAY"] = config.get_display()
self._runtime.startTime = dt.isoformat(dt.utcnow())
- self._resmon.start()
+ if self._resmon:
+ self._resmon.start()
# TODO: Perhaps clean-up path and ensure it exists?
os.chdir(self._runtime.cwd)
return self._runtime
@@ -88,8 +89,9 @@ def __exit__(self, exc_type, exc_value, exc_tb):
timediff.days * 86400 + timediff.seconds + timediff.microseconds / 1e6
)
# Collect monitored data
- for k, v in self._resmon.stop().items():
- setattr(self._runtime, k, v)
+ if self._resmon:
+ for k, v in self._resmon.stop().items():
+ setattr(self._runtime, k, v)
os.chdir(self._runtime.prevcwd)
diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py
index 324c6701ea..266affcaec 100644
--- a/nipype/utils/profiler.py
+++ b/nipype/utils/profiler.py
@@ -24,24 +24,6 @@
_MB = 1024.0**2
-class ResourceMonitorMock:
- """A mock class to use when the monitor is disabled."""
-
- @property
- def fname(self):
- """Get/set the internal filename"""
- return None
-
- def __init__(self, pid, freq=5, fname=None, cwd=None):
- pass
-
- def start(self):
- pass
-
- def stop(self):
- return {}
-
-
class ResourceMonitor(threading.Thread):
"""
A ``Thread`` to monitor a specific PID with a certain frequence
From 4b17465201dff5bbc117ef72dc9af57621933d34 Mon Sep 17 00:00:00 2001
From: Oscar Esteban <code@oscaresteban.es>
Date: Sun, 17 Apr 2022 21:49:45 +0200
Subject: [PATCH 9/9] fix: revise configuration of resource monitor at the
interface level
---
nipype/interfaces/base/core.py | 16 ++++++----------
nipype/interfaces/base/support.py | 11 ++++++-----
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py
index 69d621bbc1..781ba2ff23 100644
--- a/nipype/interfaces/base/core.py
+++ b/nipype/interfaces/base/core.py
@@ -173,7 +173,7 @@ class BaseInterface(Interface):
_additional_metadata = []
_redirect_x = False
_references = []
- resource_monitor = True # Enabled for this interface IFF enabled in the config
+ resource_monitor = None # Enabled for this interface IFF enabled in the config
_etelemetry_version_data = None
def __init__(
@@ -202,9 +202,10 @@ def __init__(
self.inputs.trait_set(**inputs)
self.ignore_exception = ignore_exception
-
- if resource_monitor is not None:
- self.resource_monitor = resource_monitor
+ self.resource_monitor = (
+ resource_monitor if resource_monitor is not None
+ else config.resource_monitor
+ )
if from_file is not None:
self.load_inputs_from_json(from_file, overwrite=True)
@@ -376,18 +377,13 @@ def run(self, cwd=None, ignore_exception=None, **inputs):
if successful, results
"""
- rtc = RuntimeContext(
- resource_monitor=config.resource_monitor and self.resource_monitor,
- ignore_exception=ignore_exception
- if ignore_exception is not None
- else self.ignore_exception,
- )
with indirectory(cwd or os.getcwd()):
self.inputs.trait_set(**inputs)
self._check_mandatory_inputs()
self._check_version_requirements(self.inputs)
+ rtc = RuntimeContext()
with rtc(self, cwd=cwd, redirect_x=self._redirect_x) as runtime:
# Grab inputs now, as they should not change during execution
diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py
index 85e6869bac..d2c4ba7cc2 100644
--- a/nipype/interfaces/base/support.py
+++ b/nipype/interfaces/base/support.py
@@ -31,20 +31,21 @@ class RuntimeContext(AbstractContextManager):
__slots__ = ("_runtime", "_resmon", "_ignore_exc")
- def __init__(self, resource_monitor=False, ignore_exception=False):
+ def __init__(self):
"""Initialize the context manager object."""
- self._ignore_exc = ignore_exception
- self._resmon = resource_monitor
+ self._ignore_exc = False
+ self._resmon = None
- def __call__(self, interface, cwd=None, redirect_x=False):
+ def __call__(self, interface, cwd=None, redirect_x=False, ignore_exception=False):
"""Generate a new runtime object."""
# Tear-up: get current and prev directories
_syscwd = rgetcwd(error=False) # Recover when wd does not exist
if cwd is None:
cwd = _syscwd
- if self._resmon:
+ self._ignore_exc = ignore_exception or interface.ignore_exception
+ if interface.resource_monitor is True:
_proc_pid = os.getpid()
self._resmon = ResourceMonitor(
_proc_pid,