Merge pull request #176 from VUIIS/v0.8.rc1

v0.8.rc1

MANIFEST.in

@@ -1 +1,4 @@
-include README.rst
+include README.rst LICENSE
+include dax_settings.ini
+
+graft docs

bin/dax_tools/dax

@@ -123,6 +123,9 @@ option if you use different XNAT_HOST).'
                                help='Logs file path if needed.', default=None)
     upload_parser.add_argument('--nodebug', dest='debug', action='store_false',
                                help='Avoid printing DEBUG information.')
+    upload_parser.add_argument('--nolock', dest='uselocking',
+                               action='store_false',
+                               help='Disable use of locking flag file.')
 
     # test:
     test_desc = "Test any dax files that the user created (processor.py/\
@@ -190,7 +193,7 @@ cluster section\n.')
         dax_tools.upload_tasks(
             args.logfile, args.debug, args.upload_settings, args.host,
             args.username, args.password, args.projects, args.suffix,
-            args.emailaddress)
+            args.emailaddress, args.uselocking)
 
     elif args.command == 'test':
         dax_tools.testing(args.test_file, args.project, args.sessions,

bin/old_tools/dax_upload

@@ -60,6 +60,8 @@ and errors.')
                     help='Logs file path if needed.', default=None)
     ap.add_argument('--nodebug', dest='debug', action='store_false',
                     help='Avoid printing DEBUG information.')
+    ap.add_argument('--nolock', dest='uselocking', action='store_false',
+                    help='Disable use of locking flag file.')
     return ap.parse_args()
 
 
@@ -69,4 +71,5 @@ if __name__ == '__main__':
     print('Deprecated executable. Use dax upload instead.')
     dax_tools.upload_tasks(args.logfile, args.debug, args.upload_settings,
                            args.host, args.username, args.password,
-                           args.projects, args.suffix, args.emailaddress)
+                           args.projects, args.suffix, args.emailaddress,
+                           args.uselocking)

dax/XnatUtils.py

@@ -32,9 +32,8 @@
 import collections
 import csv
 from datetime import datetime
-from dicom.dataset import Dataset, FileDataset
-import dicom
-import dicom.UID
+from pydicom.dataset import Dataset, FileDataset
+import pydicom
 import fnmatch
 import getpass
 import glob
@@ -1131,8 +1130,8 @@ def list_assessors(intf, projectid, subjectid, sessionid):
             anew['project_id'] = projectid
             anew['project_label'] = projectid
             anew['subject_id'] = asse['xnat:imagesessiondata/subject_id']
-            anew['session_id'] = asse['xnat:imagesessiondata/id']
-            anew['session_label'] = asse['xnat:imagesessiondata/label']
+            anew['session_id'] = asse['session_ID']
+            anew['session_label'] = asse['session_label']
             anew['procstatus'] = asse['%s/procstatus' % pfix]
             anew['proctype'] = asse['%s/proctype' % pfix]
             anew['qcstatus'] = asse['%s/validation/status' % pfix]
@@ -2195,6 +2194,9 @@ def upload_file_to_obj(filepath, resource_obj, remove=False, removeall=False,
     if not os.path.isfile(filepath):  # Check existence of the file
         err = "%s: file %s doesn't exist."
         raise XnatUtilsError(err % ('upload_file_to_obj', filepath))
+    elif os.path.getsize(filepath) == 0:  # Check for empty file
+        err = "%s: empty file, not uploading %s."
+        raise XnatUtilsError(err % ('upload_file_to_obj', filepath))
     else:
         # Remove previous resource to upload the new one
         if removeall and resource_obj.exists():
@@ -3553,7 +3555,7 @@ def order_dicoms(folder):
         raise XnatUtilsError('Folder not found: %s' % folder)
     dcm_files = dict()
     for dc in glob.glob(os.path.join(folder, '*.dcm')):
-        dst = dicom.read_file(dc)
+        dst = pydicom.read_file(dc)
         dcm_files[float(dst.SliceLocation)] = dc
     return collections.OrderedDict(sorted(dcm_files.items()))
 
@@ -3660,7 +3662,7 @@ def convert_nifti_2_dicoms(nifti_path, dicom_targets, dicom_source,
     # Load dicom headers
     if not os.path.isfile(dicom_source):
         raise XnatUtilsError("DICOM File %s not found ." % dicom_source)
-    adc_dcm_obj = dicom.read_file(dicom_source)
+    adc_dcm_obj = pydicom.read_file(dicom_source)
 
     # Make output_folder:
     if not os.path.exists(output_folder):
@@ -3678,7 +3680,7 @@ def convert_nifti_2_dicoms(nifti_path, dicom_targets, dicom_source,
         # Load dicom headers
         if not os.path.isfile(dcm_file):
             raise XnatUtilsError("DICOM File %s not found." % dcm_file)
-        t2_dcm_obj = dicom.read_file(dcm_file)
+        t2_dcm_obj = pydicom.read_file(dcm_file)
         dcm_obj_sorted[t2_dcm_obj.InstanceNumber] = t2_dcm_obj
 
     for vol_i in range(f_img_data.shape[2]):

dax/bin.py

@@ -210,18 +210,30 @@ def read_yaml_settings(yaml_file, logger):
     # Set Inputs from Yaml
     check_default_keys(yaml_file, doc)
 
-    # Set attributs for settings:
+    # Set attributes for settings
     attrs = doc.get('attrs')
 
-    # Read modules and processors:
+    # Set singularity image dir
+    singularity_imagedir = doc.get('singularity_imagedir')
+
+    # Read modules
+    modulelib = doc.get('modulelib')
     mods = dict()
     modules = doc.get('modules', list())
     for mod_dict in modules:
         if mod_dict.get('filepath') is None:
             err = 'Filepath not set for {}'.format(mod_dict.get('name'))
             raise DaxError(err)
+
+        mod_path = mod_dict.get('filepath')
+        if not os.path.isabs(mod_path) and modulelib:
+            # Preprend lib location
+            mod_path = os.path.join(modulelib, mod_path)
+
         mods[mod_dict.get('name')] = load_from_file(
-            mod_dict.get('filepath'), mod_dict.get('arguments'), logger)
+            mod_path, mod_dict.get('arguments'), logger)
+
+    # Read processors
     procs = dict()
     processors = doc.get('processors', list())
     for proc_dict in processors:
@@ -230,16 +242,26 @@ def read_yaml_settings(yaml_file, logger):
             raise DaxError(err)
         procs[proc_dict.get('name')] = load_from_file(
             proc_dict.get('filepath'), proc_dict.get('arguments'), logger)
+
+    # Read yaml processors
+    processorlib = doc.get('processorlib')
     yamlprocs = dict()
     yamls = doc.get('yamlprocessors', list())
     for yaml_dict in yamls:
         if yaml_dict.get('filepath') is None:
             err = 'Filepath not set for {}'.format(yaml_dict.get('name'))
             raise DaxError(err)
+
+        yaml_path = yaml_dict.get('filepath')
+        if not os.path.isabs(yaml_path) and processorlib:
+            # Preprend lib location
+            yaml_path = os.path.join(processorlib, yaml_path)
+
         yamlprocs[yaml_dict.get('name')] = load_from_file(
-            yaml_dict.get('filepath'), yaml_dict.get('arguments'), logger)
+            yaml_path, yaml_dict.get('arguments'),
+            logger, singularity_imagedir)
 
-    # project:
+    # Read projects
     proj_mod = dict()
     proj_proc = dict()
     yaml_proc = dict()
@@ -301,7 +323,7 @@ def raise_yaml_error_if_no_key(doc, yaml_file, key):
         raise DaxError(err.format(yaml_file, key))
 
 
-def load_from_file(filepath, args, logger):
+def load_from_file(filepath, args, logger, singularity_imagedir=None):
     """
     Check if a file exists and if it's a python file
     :param filepath: path to the file to test
@@ -331,6 +353,6 @@ def load_from_file(filepath, args, logger):
         logger.err(err.format(filepath))
 
     elif filepath.endswith('.yaml'):
-        return processors.AutoProcessor(filepath, args)
+        return processors.load_from_yaml(filepath, args, singularity_imagedir)
 
     return None

dax/cluster.py

@@ -252,7 +252,8 @@ class PBS(object):   # The script file generator class
     """ PBS class to generate/submit the cluster file to run a task """
     def __init__(self, filename, outfile, cmds, walltime_str, mem_mb=2048,
                  ppn=1, env=None, email=None,
-                 email_options=DAX_SETTINGS.get_email_opts(), xnat_host=None):
+                 email_options=DAX_SETTINGS.get_email_opts(), xnat_host=None,
+                 job_template=None):
         """
         Entry point for the PBS class
 
@@ -276,6 +277,7 @@ def __init__(self, filename, outfile, cmds, walltime_str, mem_mb=2048,
         self.email = email
         self.email_options = email_options
         self.ppn = ppn
+        self.job_template = job_template
         if env:
             self.env = env
         else:
@@ -308,7 +310,7 @@ def write(self):
                     'xnat_host': self.xnat_host}
 
         with open(self.filename, 'w') as f_obj:
-            f_obj.write(DAX_SETTINGS.get_job_template()
+            f_obj.write(DAX_SETTINGS.get_job_template(self.job_template)
                                     .safe_substitute(job_data))
 
     def submit(self, outlog=None, force_no_qsub=False):

dax/dax_settings.py

@@ -507,7 +507,7 @@ def get_job_extension_file(self):
         """
         return self.get('cluster', 'job_extension_file')
 
-    def get_job_template(self):
+    def get_job_template(self, filepath=None):
         """Get the job_template value from the cluster section.
 
         NOTE: This should be a relative path to a file up a directory
@@ -516,7 +516,12 @@ def get_job_template(self):
         :raise: OSError if the field is empty or if the file doesn't exist
         :return: Template class of the file containing the command
         """
-        filepath = self.get('cluster', 'job_template')
+        if filepath is None:
+            filepath = self.get('cluster', 'job_template')
+        elif not os.path.isabs(filepath):
+            # If only filename, we assume it is same folder as default
+            def_filepath = self.get('cluster', 'job_template')
+            filepath = os.path.join(os.path.dirname(def_filepath), filepath)
         if filepath is None:
             return ''
         if filepath.startswith('~/'):

dax/dax_tools_utils.py

@@ -257,6 +257,8 @@ def complete(text, state):
 uname -a # outputs node info (name, date&time, type, OS, etc)
 export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=${job_ppn} #set the variable \
 to use only the right amount of ppn
+export OMP_NUM_THREADS=${job_ppn} #as previous line for openmp code
+source ${job_env} #source the specified environement file
 SCREEN=$$$$$$$$
 SCREEN=${SCREEN:0:8}
 echo 'Screen display number for xvfb-run' $SCREEN
@@ -294,6 +296,8 @@ def complete(text, state):
 uname -a # outputs node info (name, date&time, type, OS, etc)
 export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=${job_ppn} #set the variable \
 to use only the right amount of ppn
+export OMP_NUM_THREADS=${job_ppn} #as previous line for openmp code
+source ${job_env} #source the specified environement file
 SCREEN=$$$$$$$$
 SCREEN=${SCREEN:0:8}
 echo 'Screen display number for xvfb-run' $SCREEN
@@ -336,6 +340,8 @@ def complete(text, state):
 uname -a # outputs node info (name, date&time, type, OS, etc)
 export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=${job_ppn} #set the variable \
 to use only the right amount of ppn
+export OMP_NUM_THREADS=${job_ppn} #as previous line for openmp code
+source ${job_env} #source the specified environement file
 SCREEN=$$$$$$$$
 SCREEN=${SCREEN:0:8}
 echo 'Screen display number for xvfb-run' $SCREEN
@@ -456,11 +462,12 @@ def complete(text, state):
         memory: {memory}
         walltime: {walltime}
         Number of cores: {ppn}
+        Environment file: {env}
       OTHER ARGUMENTS:
 {other}
 """
 PROC_DEF_ARGS = ['name', 'xnat_host', 'xsitype', 'memreq_mb', 'walltime_str',
-                 'ppn', 'spider_path', 'version']
+                 'ppn', 'env', 'spider_path', 'version']
 
 MOD_DISPLAY = """    *NAME: {name}
       TEMP DIRECTORY: {temp_dir}
@@ -480,7 +487,8 @@ def complete(text, state):
 
 def upload_tasks(logfile, debug, upload_settings=None,
                  host=None, username=None, password=None,
-                 projects=None, suffix=None, emailaddress=None):
+                 projects=None, suffix=None, emailaddress=None,
+                 uselocking=True):
     """
     Upload tasks from the queue folder.
 
@@ -500,21 +508,22 @@ def upload_tasks(logfile, debug, upload_settings=None,
 
     # Check if folders exist
     check_folders()
-    ##flagfile = "%s%s.txt" % (FLAGFILE_TEMPLATE, suffix)
+    flagfile = "%s%s.txt" % (FLAGFILE_TEMPLATE, suffix)
 
     # Load the settings for upload
     upload_settings = load_upload_settings(upload_settings, host, username,
                                            password, projects)
     print_upload_settings(upload_settings)
     # create the flag file showing that the spider is running
-    ##if is_dax_upload_running(flagfile):
-    ##    pass
-    ##else:
-    ##    try:
-    upload_results(upload_settings, emailaddress)
-    ##    finally:
-            # remove flagfile
-    ##        os.remove(flagfile)
+    if uselocking and is_dax_upload_running(flagfile):
+        pass
+    else:
+        try:
+            upload_results(upload_settings, emailaddress)
+        finally:
+            if uselocking:
+                # remove flagfile
+                os.remove(flagfile)
 
 
 def testing(test_file, project, sessions, host=None, username=None, hide=False,
@@ -1077,6 +1086,7 @@ def upload_snapshots(assessor_obj, resource_path):
         assessor_obj.out_resource('SNAPSHOTS').delete()
     original = os.path.join(resource_path, SNAPSHOTS_ORIGINAL)
     thumbnail = os.path.join(resource_path, SNAPSHOTS_PREVIEW)
+    status = None
     try:
         status = XnatUtils.upload_assessor_snapshots(
             assessor_obj, original, thumbnail)
@@ -1329,11 +1339,12 @@ def load_upload_settings(f_settings, host, username, password, projects):
         _host = os.environ['XNAT_HOST']
         username = None
         password = None
-        projects = []
         if host:
             _host = host
         if projects:
             projects = projects.split(',')
+        else:
+            projects = []
         if username:
             username = username
             if not password:
@@ -2061,6 +2072,7 @@ def print_processor(proc_obj):
                               memory=proc_dict['memreq_mb'],
                               walltime=proc_dict['walltime_str'],
                               ppn=proc_dict['ppn'],
+                              env=proc_dict['env'],
                               other=other_args))
 
 
@@ -2152,7 +2164,7 @@ def load_test(filepath):
         else:
             # So far only auto processor:
             try:
-                return processors.AutoProcessor(filepath)
+                return processors.load_from_yaml(filepath)
             except AutoProcessorError:
                 print('[ERROR]')
                 exc_type, exc_value, exc_traceback = sys.exc_info()

dax/launcher.py

@@ -129,7 +129,7 @@ def __init__(self,
                 if isinstance(yaml_obj, processors.AutoProcessor):
                     proc = yaml_obj
                 else:
-                    proc = processors.AutoProcessor(yaml_obj)
+                    proc = processors.load(yaml_obj)
                 if project not in self.project_process_dict:
                     self.project_process_dict[project] = [proc]
                 else: