Merge 1.5.1 Release into master

Author: sean-smith

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+*Issue #, if available:*
+
+*Description of changes:*
+
+
+By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

CHANGELOG.md

@@ -0,0 +1,12 @@
+cfncluster-node CHANGELOG
+=========================
+
+This file is used to list changes made in each version of the cfncluster-node package.
+
+1.5.1
+-----
+
+Bug fixes/minor improvements:
+
+  - Fixed Torque behaviour when scaling up from an empty cluster
+  - Avoid Torque server restart when adding and removing compute nodes

CODE_OF_CONDUCT.md

@@ -0,0 +1,4 @@
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 
+opensource-codeofconduct@amazon.com with any additional questions or comments.

CONTRIBUTING.md

@@ -0,0 +1,61 @@
+# Contributing Guidelines
+
+Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 
+documentation, we greatly value feedback and contributions from our community.
+
+Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 
+information to effectively respond to your bug report or contribution.
+
+
+## Reporting Bugs/Feature Requests
+
+We welcome you to use the GitHub issue tracker to report bugs or suggest features.
+
+When filing an issue, please check [existing open](https://github.com/awslabs/cfncluster-node/issues), or [recently closed](https://github.com/awslabs/cfncluster-node/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 
+reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
+
+* A reproducible test case or series of steps
+* The version of our code being used
+* Any modifications you've made relevant to the bug
+* Anything unusual about your environment or deployment
+
+
+## Contributing via Pull Requests
+Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
+
+1. You are working against the latest source on the *develop* branch.
+2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
+3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
+
+To send us a pull request, please:
+
+1. Fork the repository.
+2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
+3. Ensure local tests pass.
+4. Commit to your fork using clear commit messages.
+5. Send us a pull request, answering any default questions in the pull request interface.
+6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
+
+GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 
+[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
+
+
+## Finding contributions to work on
+Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/cfncluster-node/labels/help%20wanted) issues is a great place to start. 
+
+
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 
+opensource-codeofconduct@amazon.com with any additional questions or comments.
+
+
+## Security issue notifications
+If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
+
+
+## Licensing
+
+See the [LICENSE](https://github.com/awslabs/cfncluster-node/blob/develop/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+
+We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.

nodewatcher/plugins/torque.py

@@ -41,7 +41,7 @@ def getJobs(hostname):
     try:
         status, output = runPipe(commands)
     except subprocess.CalledProcessError:
-        log.error("Failed to run %s\n" % _command)
+        log.error("Failed to run %s\n" % commands)
 
     if output == "":
         _jobs = False

setup.py

@@ -21,7 +21,7 @@ def read(fname):
 
 console_scripts = ['sqswatcher = sqswatcher.sqswatcher:main', 
                    'nodewatcher = nodewatcher.nodewatcher:main']
-version = "1.4.3"
+version = "1.5.1"
 requires = ['boto>=2.48.0', 'python-dateutil>=2.6.1']
 
 if sys.version_info[:2] == (2, 6):

sqswatcher/plugins/torque.py

@@ -16,17 +16,68 @@
 import paramiko
 import logging
 import shlex
+import time
+import xml.etree.ElementTree as xmltree
+import socket
 
 log = logging.getLogger(__name__)
 
 def __runCommand(command):
     log.debug(repr(command))
     _command = shlex.split(str(command))
     log.debug(_command)
+
+    DEV_NULL = open(os.devnull, "rb")
     try:
-        sub.check_call(_command, env=dict(os.environ))
-    except sub.CalledProcessError:
-        log.error("Failed to run %s\n" % _command)
+        process = sub.Popen(_command, env=dict(os.environ), stdout=sub.PIPE, stderr=sub.STDOUT, stdin=DEV_NULL)
+        stdout = process.communicate()[0]
+        exitcode = process.poll()
+        if exitcode != 0:
+            log.error("Failed to run %s:\n%s" % (_command, stdout))
+        return stdout
+    finally:
+        DEV_NULL.close()
+
+
+def isHostInitState(host_state):
+    # Node states http://docs.adaptivecomputing.com/torque/6-0-2/adminGuide/help.htm#topics/torque/8-resources/resources.htm#nodeStates
+    init_states = ("down", "offline", "unknown", str(None))
+    return str(host_state).startswith(init_states)
+
+def wakeupSchedOn(hostname):
+    log.info('Waking up scheduler on host %s', hostname)
+    command = ("/opt/torque/bin/pbsnodes -x %s" % (hostname))
+
+    sleep_time = 3
+    times = 20
+    host_state = None
+    while isHostInitState(host_state) and times > 0:
+        output = __runCommand(command)
+        try:
+            # Ex.1: <Data><Node><name>ip-10-0-76-39</name><state>down,offline,MOM-list-not-sent</state><power_state>Running</power_state>
+            #        <np>1</np><ntype>cluster</ntype><mom_service_port>15002</mom_service_port><mom_manager_port>15003</mom_manager_port></Node></Data>
+            # Ex 2: <Data><Node><name>ip-10-0-76-39</name><state>free</state><power_state>Running</power_state><np>1</np><ntype>cluster</ntype>
+            #        <status>rectime=1527799181,macaddr=02:e4:00:b0:b1:72,cpuclock=Fixed,varattr=,jobs=,state=free,netload=210647044,gres=,loadave=0.00,
+            #        ncpus=1,physmem=1017208kb,availmem=753728kb,totmem=1017208kb,idletime=856,nusers=1,nsessions=1,sessions=19698,
+            #        uname=Linux ip-10-0-76-39 4.9.75-25.55.amzn1.x86_64 #1 SMP Fri Jan 5 23:50:27 UTC 2018 x86_64,opsys=linux</status>
+            #        <mom_service_port>15002</mom_service_port><mom_manager_port>15003</mom_manager_port></Node></Data>
+            xmlnode = xmltree.XML(output)
+            host_state = xmlnode.findtext("./Node/state")
+        except:
+            log.error("Error parsing XML from %s" % output)
+
+        if isHostInitState(host_state):
+            log.debug("Host %s is still in state %s" % (hostname, host_state))
+            time.sleep(sleep_time)
+            times -= 1
+
+    if host_state == "free":
+        command = "/opt/torque/bin/qmgr -c \"set server scheduling=true\""
+        __runCommand(command)
+    elif times == 0:
+        log.error("Host %s is still in state %s" % (hostname, host_state))
+    else:
+        log.debug("Host %s is in state %s" % (hostname, host_state))
 
 def addHost(hostname,cluster_user,slots):
     log.info('Adding %s', hostname)
@@ -64,8 +115,7 @@ def addHost(hostname,cluster_user,slots):
     ssh.save_host_keys(hosts_key_file)
     ssh.close()
 
-    command = ('/etc/init.d/pbs_server restart')
-    __runCommand(command)
+    wakeupSchedOn(hostname)
 
 def removeHost(hostname, cluster_user):
     log.info('Removing %s', hostname)
@@ -76,6 +126,3 @@ def removeHost(hostname, cluster_user):
     command = ("/opt/torque/bin/qmgr -c 'delete node %s'" % hostname)
     __runCommand(command)
 
-    command = ('/etc/init.d/pbs_server restart')
-    __runCommand(command)
-