Merge pull request #1 from boegel/pgi

sync with develop + fix remarks for support for CrayPGI toolchain

Author: jgphpc

.travis.yml

@@ -0,0 +1,75 @@
+language: python
+python: 2.6
+env:
+  matrix:
+    # purposely specifying slowest builds first, to gain time overall
+    - LMOD_VERSION=5.6.3 TEST_EASYBUILD_MODULES_TOOL=Lmod
+    - LMOD_VERSION=6.3.1 TEST_EASYBUILD_MODULES_TOOL=Lmod
+    - LMOD_VERSION=6.3.1 TEST_EASYBUILD_MODULES_TOOL=Lmod TEST_EASYBUILD_MODULE_SYNTAX=Lua
+    - ENV_MOD_VERSION=3.2.10
+    - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl
+matrix:
+  # mark build as finished as soon as job has failed
+  fast_finish: true
+  include:
+    # also test default configuration with Python 2.7
+    - python: 2.7
+      env: ENV_MOD_VERSION=3.2.10
+addons:
+  apt:
+    packages:
+      # for environment modules/Lmod
+      - tcl8.5
+      # for EasyBuild
+      - python-setuptools
+      # for GitPython, python-hglib
+      - git
+      - mercurial
+      # for GC3Pie (optional dep for EasyBuild)
+      - time
+before_install:
+    # keyring is required to provide GitHub token to EasyBuild;
+    # keyring v5.7.1 is last version to be compatible with py2.6;
+    # for recent versions of keyring, keyrings.alt must be installed too
+    - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install keyring==5.7.1; else pip install keyring keyrings.alt; fi
+    # optional Python packages for EasyBuild
+    - pip install autopep8 GC3Pie GitPython python-graph-dot python-hglib PyYAML
+    # git config is required to make actual git commits (cfr. tests for GitRepository)
+    - git config --global user.name "Travis CI"
+    - git config --global user.email "travis@travis-ci.org"
+install:
+    # install vsc-base (& vsc-install) dependencies for EasyBuild
+    - easy_install vsc-base
+    # install environment modules or Lmod
+    - export INSTALL_DEP=$TRAVIS_BUILD_DIR/easybuild/scripts/install_eb_dep.sh
+    - if [ ! -z $ENV_MOD_VERSION ]; then source $INSTALL_DEP modules-${ENV_MOD_VERSION} $HOME; fi
+    - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP lua-5.1.4.8 $HOME; fi
+    - if [ ! -z $LMOD_VERSION ]; then source $INSTALL_DEP Lmod-${LMOD_VERSION} $HOME; fi
+    - if [ ! -z $ENV_MOD_TCL_VERSION ]; then source $INSTALL_DEP modules-tcl-${ENV_MOD_TCL_VERSION} $HOME; fi
+script:
+    # set up environment for modules tool (if $MOD_INIT is defined)
+    - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; fi
+    # set up environment for EasyBuild framework tests
+    - export PATH=$TRAVIS_BUILD_DIR:$PATH
+    - export PYTHONPATH=$TRAVIS_BUILD_DIR
+    # install GitHub token
+    - if [ ! -z $GITHUB_TOKEN ]; then
+        if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ];
+            then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
+            else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
+        fi;
+        python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+      fi
+    # run test suite
+    - python -O -m test.framework.suite
+    # configure EasyBuild before running bootstrap (modules tool/syntax to use)
+    - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi
+    - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi
+    # move outside of checkout of easybuild-framework repository, weird place to run bootstrap
+    - cd $HOME
+    # test bootstrap script
+    - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID
+    # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
+    - unset PYTHONPATH
+    # simply sanity check on bootstrapped EasyBuild module
+    - module use /tmp/$TRAVIS_JOB_ID/modules/all; module load EasyBuild; eb --version

easybuild/__init__.py

@@ -28,14 +28,5 @@
 @author: Jens Timmerman (Ghent University)
 @author: Kenneth Hoste (Ghent University)
 """
-import os
 import pkg_resources
-import sys
-
-# check whether EasyBuild is being run from a directory that contains easybuild/__init__.py;
-# that doesn't work (fails with import errors), due to weirdness to Python packaging/setuptools/namespaces
-if __path__[0] == 'easybuild':
-    sys.stderr.write("ERROR: Running EasyBuild from %s does not work (Python packaging weirdness)...\n" % os.getcwd())
-    sys.exit(1)
-
 pkg_resources.declare_namespace(__name__)

easybuild/framework/easyblock.py

@@ -64,16 +64,16 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import DEFAULT_CHECKSUM
-from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, download_file, encode_class_name
-from easybuild.tools.filetools import extract_file, mkdir, move_logs, read_file, rmtree2
-from easybuild.tools.filetools import write_file, compute_checksum, verify_checksum, weld_paths
+from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, derive_alt_pypi_url
+from easybuild.tools.filetools import download_file, encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs
+from easybuild.tools.filetools import read_file, rmtree2, write_file, compute_checksum, verify_checksum, weld_paths
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
-from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name
-from easybuild.tools.modules import get_software_root, modules_tool
+from easybuild.tools.modules import invalidate_module_caches_for, get_software_root, get_software_root_env_var_name
+from easybuild.tools.modules import get_software_version_env_var_name
 from easybuild.tools.package.utilities import package
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
@@ -101,6 +101,9 @@
 
 MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP]
 
+# string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
+PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/'
+
 
 _log = fancylogger.getLogger('easyblock')
 
@@ -154,8 +157,14 @@ def __init__(self, ec):
         self.skip = None
         self.module_extra_extensions = ''  # extra stuff for module file required by extensions
 
+        # easyconfig for this application
+        if isinstance(ec, EasyConfig):
+            self.cfg = ec
+        else:
+            raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec)
+
         # modules interface with default MODULEPATH
-        self.modules_tool = modules_tool()
+        self.modules_tool = self.cfg.modules_tool
         # module generator
         self.module_generator = module_generator(self, fake=True)
 
@@ -170,12 +179,6 @@ def __init__(self, ec):
         if modules_header_path is not None:
             self.modules_header = read_file(modules_header_path)
 
-        # easyconfig for this application
-        if isinstance(ec, EasyConfig):
-            self.cfg = ec
-        else:
-            raise EasyBuildError("Value of incorrect type passed to EasyBlock constructor: %s ('%s')", type(ec), ec)
-
         # determine install subdirectory, based on module name
         self.install_subdir = None
 
@@ -622,6 +625,16 @@ def obtain_file(self, filename, extension=False, urls=None):
                         self.log.warning("Source URL %s is of unknown type, so ignoring it." % url)
                         continue
 
+                    # PyPI URLs may need to be converted due to change in format of these URLs,
+                    # cfr. https://bitbucket.org/pypa/pypi/issues/438
+                    if PYPI_PKG_URL_PATTERN in fullurl and not is_alt_pypi_url(fullurl):
+                        alt_url = derive_alt_pypi_url(fullurl)
+                        if alt_url:
+                            _log.debug("Using alternate PyPI URL for %s: %s", fullurl, alt_url)
+                            fullurl = alt_url
+                        else:
+                            _log.debug("Failed to derive alternate PyPI URL for %s, so retaining the original", fullurl)
+
                     if self.dry_run:
                         self.dry_run_msg("  * %s will be downloaded to %s", filename, targetpath)
                         if extension and urls:
@@ -693,6 +706,13 @@ def short_mod_name(self):
         """
         return self.cfg.short_mod_name
 
+    @property
+    def mod_subdir(self):
+        """
+        Subdirectory in module install path
+        """
+        return self.cfg.mod_subdir
+
     @property
     def moduleGenerator(self):
         """
@@ -905,7 +925,7 @@ def make_module_dep(self, unload_info=None):
         self.log.debug("Full list of dependencies: %s" % deps)
 
         # exclude dependencies that extend $MODULEPATH and form the path to the top of the module tree (if any)
-        full_mod_subdir = os.path.join(self.installdir_mod, self.cfg.mod_subdir)
+        full_mod_subdir = os.path.join(self.installdir_mod, self.mod_subdir)
         init_modpaths = mns.det_init_modulepaths(self.cfg)
         top_paths = [self.installdir_mod] + [os.path.join(self.installdir_mod, p) for p in init_modpaths]
         excluded_deps = self.modules_tool.path_to_top_of_module_tree(top_paths, self.cfg.short_mod_name,
@@ -1132,7 +1152,14 @@ def load_module(self, mod_paths=None, purge=True):
             if mod_paths is None:
                 mod_paths = []
             all_mod_paths = mod_paths + ActiveMNS().det_init_modulepaths(self.cfg)
-            mods = [self.full_mod_name]
+
+            # for flat module naming schemes, we can load the module directly;
+            # for non-flat (hierarchical) module naming schemes, we may need to load the toolchain module first
+            # to update $MODULEPATH such that the module can be loaded using the short module name
+            mods = [self.short_mod_name]
+            if self.mod_subdir and self.toolchain.name != DUMMY_TOOLCHAIN_NAME:
+                mods.insert(0, self.toolchain.det_short_module_name())
+
             self.modules_tool.load(mods, mod_paths=all_mod_paths, purge=purge, init_env=self.initial_environ)
         else:
             self.log.warning("Not loading module, since self.full_mod_name is not set.")
@@ -1148,7 +1175,7 @@ def load_fake_module(self, purge=False):
         fake_mod_path = self.make_module_step(fake=True)
 
         # load fake module
-        self.modules_tool.prepend_module_path(fake_mod_path)
+        self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir))
         self.load_module(purge=purge)
 
         return (fake_mod_path, env)
@@ -1159,16 +1186,16 @@ def clean_up_fake_module(self, fake_mod_data):
         """
         fake_mod_path, env = fake_mod_data
         # unload module and remove temporary module directory
-        # self.full_mod_name might not be set (e.g. during unit tests)
-        if fake_mod_path and self.full_mod_name is not None:
+        # self.short_mod_name might not be set (e.g. during unit tests)
+        if fake_mod_path and self.short_mod_name is not None:
             try:
-                self.modules_tool.unload([self.full_mod_name])
-                self.modules_tool.remove_module_path(fake_mod_path)
+                self.modules_tool.unload([self.short_mod_name])
+                self.modules_tool.remove_module_path(os.path.join(fake_mod_path, self.mod_subdir))
                 rmtree2(os.path.dirname(fake_mod_path))
             except OSError, err:
                 raise EasyBuildError("Failed to clean up fake module dir %s: %s", fake_mod_path, err)
-        elif self.full_mod_name is None:
-            self.log.warning("Not unloading module, since self.full_mod_name is not set.")
+        elif self.short_mod_name is None:
+            self.log.warning("Not unloading module, since self.short_mod_name is not set.")
 
         # restore original environment
         restore_env(env)
@@ -1581,6 +1608,9 @@ def extensions_step(self, fetch=False):
         if not self.dry_run:
             fake_mod_data = self.load_fake_module(purge=True)
 
+            # also load modules for build dependencies again, since those are not loaded by the fake module
+            self.modules_tool.load(dep['short_mod_name'] for dep in self.cfg['builddependencies'])
+
         self.prepare_for_extensions()
 
         if fetch:
@@ -1666,6 +1696,17 @@ def extensions_step(self, fetch=False):
                 msg = "\n* installing extension %s %s using '%s' easyblock\n" % (ext['name'], ext['version'], eb_class)
                 self.dry_run_msg(msg)
 
+            self.log.debug("List of loaded modules: %s", self.modules_tool.list())
+
+            # prepare toolchain build environment, but only when not doing a dry run
+            # since in that case the build environment is the same as for the parent
+            if self.dry_run:
+                self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...")
+            else:
+                # don't reload modules for toolchain, there is no need since they will be loaded already;
+                # the (fake) module for the parent software gets loaded before installing extensions
+                inst.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False)
+
             # real work
             inst.prerun()
             txt = inst.run()
@@ -1951,10 +1992,19 @@ def make_module_step(self, fake=False):
 
         else:
             write_file(mod_filepath, txt)
-
             self.log.info("Module file %s written: %s", mod_filepath, txt)
 
-             # only update after generating final module file
+            # invalidate relevant 'module avail'/'module show' cache entries
+            modpath = self.module_generator.get_modules_path(fake=fake)
+            # consider both paths: for short module name, and subdir indicated by long module name
+            paths = [modpath]
+            if self.mod_subdir:
+                paths.append(os.path.join(modpath, self.mod_subdir))
+
+            for path in paths:
+                invalidate_module_caches_for(path)
+
+            # only update after generating final module file
             if not fake:
                 self.modules_tool.update()
 
@@ -1999,6 +2049,22 @@ def permissions_step(self):
             adjust_permissions(self.installdir, perms, add=False, recursive=True, relative=True, ignore_errors=True)
             self.log.info("Successfully removed write permissions recursively for group/other on install dir.")
 
+            # add read permissions for everybody on all files, taking into account group (if any)
+            perms = stat.S_IRUSR | stat.S_IRGRP
+            self.log.debug("Ensuring read permissions for user/group on install dir (recursively)")
+            if self.group is None:
+                perms |= stat.S_IROTH
+                self.log.debug("Also ensuring read permissions for others on install dir (no group specified)")
+
+            umask = build_option('umask')
+            if umask is not None:
+                # umask is specified as a string, so interpret it first as integer in octal, then take complement (~)
+                perms &= ~int(umask, 8)
+                self.log.debug("Taking umask '%s' into account when ensuring read permissions to install dir", umask)
+
+            adjust_permissions(self.installdir, perms, add=True, recursive=True, relative=True, ignore_errors=True)
+            self.log.info("Successfully added read permissions '%s' recursively on install dir", oct(perms))
+
     def test_cases_step(self):
         """
         Run provided test cases.

easybuild/framework/easyconfig/easyconfig.py

@@ -264,6 +264,8 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
             self.rawtxt = rawtxt
             self.log.debug("Supplied raw easyconfig contents: %s" % self.rawtxt)
 
+        self.modules_tool = modules_tool()
+
         # use legacy module classes as default
         self.valid_module_classes = build_option('valid_module_classes')
         if self.valid_module_classes is not None:
@@ -643,7 +645,8 @@ def toolchain(self):
                     tcdeps = tc_ec['dependencies']
                     self.log.debug("Toolchain dependencies based on easyconfig: %s", tcdeps)
 
-            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'], mns=ActiveMNS(), tcdeps=tcdeps)
+            self._toolchain = get_toolchain(self['toolchain'], self['toolchainopts'],
+                                            mns=ActiveMNS(), tcdeps=tcdeps, modtool=self.modules_tool)
             tc_dict = self._toolchain.as_dict()
             self.log.debug("Initialized toolchain: %s (opts: %s)" % (tc_dict, self['toolchainopts']))
         return self._toolchain
@@ -808,7 +811,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
                                dependency, tc)
                 # update the toolchain with the minimal value
                 orig_tc = tc
-                tc = robot_find_minimal_toolchain_of_dependency(dependency, parent_tc=tc)
+                tc = robot_find_minimal_toolchain_of_dependency(dependency, self.modules_tool, parent_tc=tc)
                 if tc is None:
                     raise EasyBuildError("No easyconfig for %s that matches toolchain hierarchy generated by %s",
                                          dependency, orig_tc)
@@ -1267,7 +1270,7 @@ def robot_find_easyconfig(name, version):
     return res
 
 
-def robot_find_minimal_toolchain_of_dependency(dep, parent_tc=None):
+def robot_find_minimal_toolchain_of_dependency(dep, modtool, parent_tc=None):
     """
     Find the minimal toolchain of a dependency
 
@@ -1278,7 +1281,6 @@ def robot_find_minimal_toolchain_of_dependency(dep, parent_tc=None):
     if parent_tc is None:
         parent_tc = dep['toolchain']
 
-    modtool = modules_tool()
     avail_modules = []
     if build_option('use_existing_modules') and not build_option('retain_all_deps'):
         avail_modules = modtool.available()

easybuild/framework/easyconfig/tools.py

@@ -83,9 +83,8 @@
 _log = fancylogger.getLogger('easyconfig.tools', fname=False)
 
 
-def skip_available(easyconfigs):
+def skip_available(easyconfigs, modtool):
     """Skip building easyconfigs for existing modules."""
-    modtool = modules_tool()
     module_names = [ec['full_mod_name'] for ec in easyconfigs]
     modules_exist = modtool.exist(module_names)
     retained_easyconfigs = []
@@ -98,7 +97,7 @@ def skip_available(easyconfigs):
     return retained_easyconfigs
 
 
-def find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=False):
+def find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=False):
     """
     Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
 
@@ -108,7 +107,6 @@ def find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=False):
     """
     ordered_ecs = []
     new_easyconfigs = []
-    modtool = modules_tool()
     # copy, we don't want to modify the origin list of available modules
     avail_modules = avail_modules[:]
     _log.debug("Finding resolved modules for %s (available modules: %s)", easyconfigs, avail_modules)
@@ -434,7 +432,7 @@ def find_related_easyconfigs(path, ec):
 
     regexes = []
     for version_pattern in version_patterns:
-        common_pattern = r'^\S+/%s-%s%%s\.eb$' % (name, version_pattern)
+        common_pattern = r'^\S+/%s-%s%%s\.eb$' % (re.escape(name), version_pattern)
         regexes.extend([
             common_pattern % (toolchain_pattern + versionsuffix),
             common_pattern % (toolchain_name_pattern + versionsuffix),