v0.5.4 update

various bug fixes
updated PostgreSQL dependency to 42.7.3
updated JSch dependency to 0.2.17
added random offset to stagger database queries
improved PostgreSQL authentication scheme - client certificates
improved SFTP authentication scheme - private keys
added partial two-way operator syncing from server to database
added descriptive hovertext to tables in the UI
added WebCTRL servers page to the UI
added function to save and retrieve license files from any server
added function to see all applied cumulative updates on any server
added notes column to add extra information to each WebCTRL server
added operator blacklist exceptions table to the UI
added optional add-on feature
add log expiration feature

Author: cvogt729

config/BUILD_DETAILS

@@ -1,32 +1,34 @@
 JDK Version:
-openjdk 19.0.1 2022-10-18
-OpenJDK Runtime Environment (build 19.0.1+10-21)
-OpenJDK 64-Bit Server VM (build 19.0.1+10-21, mixed mode, sharing)
+openjdk 21.0.2 2024-01-16
+OpenJDK Runtime Environment (build 21.0.2+13-58)
+OpenJDK 64-Bit Server VM (build 21.0.2+13-58, mixed mode, sharing)
 
 Compilation Flags:
---release 8
+--release 11
 
 Runtime Dependencies:
-addonsupport-api-addon-1.9.0
-alarmmanager-api-addon-1.9.0
-bacnet-api-core-1.10.002-20230209.1438r
-directaccess-api-addon-1.9.0
-tomcat-embed-core-9.0.68
-webaccess-api-addon-1.9.0
-xdatabase-api-addon-1.9.0
-common-8.5.002
-commonbaseutils-2.0.3
-commonexceptions-8.5.002
-core-8.5.002
-datatable-8.5.002
-directaccess-8.5.002
-directaccess-api-8.5.002
-extensionsupport-api-8.5.002
-jsch-0.2.11-sources
-postgresql-42.6.0-sources
-spring-context-5.2.22.RELEASE
-webserver-api-8.5.002
+addonsupport-api-addon-1.10.0
+alarmmanager-api-addon-1.10.0
+bacnet-api-core-1.10.007-20240227.1003r
+directaccess-api-addon-1.10.0
+tomcat-embed-core-9.0.87
+webaccess-api-addon-1.10.0
+xdatabase-api-addon-1.10.0
+cjupdate-1.2.4
+common-9.0.002
+commonbaseutils-2.0.5
+commonexceptions-9.0.002
+core-9.0.002
+datatable-9.0.002
+directaccess-9.0.002
+directaccess-api-9.0.002
+extensionsupport-api-9.0.002
+jsch-0.2.17-sources
+postgresql-42.7.3-sources
+spring-context-5.3.33
+update-api-9.0.002
+webserver-api-9.0.002
 
 Packaged Dependencies:
-jsch-0.2.11
-postgresql-42.6.0
+jsch-0.2.17
+postgresql-42.7.3

config/COMPILE_FLAGS

@@ -1 +1 @@
---release 8
+--release 11

config/EXTERNAL_DEPS

@@ -1,2 +1,2 @@
-url:postgresql:https://repo1.maven.org/maven2/org/postgresql/postgresql/42.6.0/postgresql-42.6.0.jar
-url:jsch:https://repo1.maven.org/maven2/com/github/mwiede/jsch/0.2.11/jsch-0.2.11.jar
+url:postgresql:https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.3/postgresql-42.7.3.jar
+url:jsch:https://repo1.maven.org/maven2/com/github/mwiede/jsch/0.2.17/jsch-0.2.17.jar

config/RUNTIME_DEPS

@@ -8,5 +8,7 @@ file:commonexceptions:modules\commonexceptions
 file:commonbaseutils:bin\lib
 file:directaccess:modules\directaccess
 file:directaccess-api:modules\directaccess
-url:postgresql:https://repo1.maven.org/maven2/org/postgresql/postgresql/42.6.0/postgresql-42.6.0-sources.jar
-url:jsch:https://repo1.maven.org/maven2/com/github/mwiede/jsch/0.2.11/jsch-0.2.11-sources.jar
+file:cjupdate:bin\lib
+file:update-api:modules\update
+url:postgresql:https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.3/postgresql-42.7.3-sources.jar
+url:jsch:https://repo1.maven.org/maven2/com/github/mwiede/jsch/0.2.17/jsch-0.2.17-sources.jar

root/info.xml

@@ -1,7 +1,7 @@
 <extension version="1">
   <name>PostgreSQL_Connect</name>
-  <description>Periodically synchronizes data to an external PostgreSQL database.</description>
-  <version>0.4.5</version>
+  <description>Periodically synchronizes operators, add-ons, and specified trends with an external PostgreSQL database.</description>
+  <version>0.5.4</version>
   <vendor>Automatic Controls Equipment Systems, Inc.</vendor>
   <system-menu-provider>aces.webctrl.postgresql.web.SystemMenuEditor</system-menu-provider>
 </extension>

root/webapp/WEB-INF/web.xml

@@ -28,6 +28,24 @@
     <url-pattern>/TableEditor</url-pattern>
   </servlet-mapping>
 
+  <servlet>
+    <servlet-name>InstallAddonPage</servlet-name>
+    <servlet-class>aces.webctrl.postgresql.web.InstallAddonPage</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>InstallAddonPage</servlet-name>
+    <url-pattern>/InstallAddon</url-pattern>
+  </servlet-mapping>
+
+  <servlet>
+    <servlet-name>DownloadLicensePage</servlet-name>
+    <servlet-class>aces.webctrl.postgresql.web.DownloadLicensePage</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>DownloadLicensePage</servlet-name>
+    <url-pattern>/DownloadLicense</url-pattern>
+  </servlet-mapping>
+
   <servlet>
     <servlet-name>SaveOperator</servlet-name>
     <servlet-class>aces.webctrl.postgresql.web.SaveOperator</servlet-class>

src/aces/webctrl/postgresql/core/AddonDownload.java

@@ -1,12 +1,33 @@
 package aces.webctrl.postgresql.core;
 import java.nio.file.*;
 import com.controlj.green.webserver.AddOn;
+import java.util.regex.*;
 public class AddonDownload {
-  public String name;
+  public String displayName;
   public String version;
   public String path;
   public boolean removeData;
   public boolean keepNewer;
+  public boolean optional;
   public Path file = null;
   public AddOn addon = null;
+  private String refname = null;
+  private final static Pattern filenameParser = Pattern.compile("[/\\\\]([^/\\\\]+)\\.addon$");
+  public String getReferenceName(){
+    if (refname!=null){
+      return refname;
+    }
+    final Matcher m = filenameParser.matcher(path);
+    if (m.find()){
+      refname = m.group(1);
+    }
+    if (refname==null){
+      if (displayName==null){
+        refname = "addon";
+      }else{
+        refname = displayName;
+      }
+    }
+    return refname;
+  }
 }

src/aces/webctrl/postgresql/core/Config.java

@@ -3,18 +3,22 @@
 import java.nio.file.*;
 import java.nio.channels.*;
 public class Config {
-  private volatile static Path file = null;
+  public volatile static Path file = null;
   public volatile static String connectionURL = null;
   public volatile static String username = null;
   public volatile static String password = null;
+  public volatile static String keystorePassword = null;
   public volatile static int ID;
   public final static CronExpression cron = new CronExpression();
+  public volatile static long maxRandomOffset;
   static { revertDefaults(); }
   private static void revertDefaults(){
     ID = -1;
     connectionURL = "";
     username = "";
     password = "";
+    keystorePassword = "";
+    maxRandomOffset = 0L;
     cron.set("0 0 * * * *");
   }
   private Config(){}
@@ -29,9 +33,8 @@ public static void reset(){
       Sync.started = false;
     }
   }
-  public static boolean init(Path file){
+  public static void init(Path file){
     Config.file = file;
-    return load();
   }
   public static boolean load(){
     if (file==null){
@@ -46,11 +49,14 @@ public static boolean load(){
         try{
           final SerializationStream s = new SerializationStream(arr);
           ID = s.readInt();
-          /*final String version =*/ s.readString();
+          //final String version = 
+            s.readString();
           connectionURL = s.readString();
           username = s.readString();
           password = s.readString();
+          maxRandomOffset = s.readLong();
           cron.set(s.readString());
+          keystorePassword = s.readString();
           if (!s.end()){
             Initializer.log("Configuration file corrupted. Parameters reverted to defaults.",true);
             revertDefaults();
@@ -80,6 +86,7 @@ public static boolean save(){
       l+=username.length();
       l+=password.length();
       l+=cronExpr.length();
+      l+=keystorePassword.length();
       l<<=1;
       l+=4;
       final SerializationStream s = new SerializationStream(l, true);
@@ -88,7 +95,9 @@ public static boolean save(){
       s.write(connectionURL);
       s.write(username);
       s.write(password);
+      s.write(maxRandomOffset);
       s.write(cronExpr);
+      s.write(keystorePassword);
       final ByteBuffer buf = s.getBuffer();
       synchronized(Config.class){
         try(

src/aces/webctrl/postgresql/core/ConnectSFTP.java

@@ -1,5 +1,7 @@
 package aces.webctrl.postgresql.core;
 import java.io.*;
+import java.nio.*;
+import java.nio.channels.*;
 import java.nio.file.*;
 import com.jcraft.jsch.*;
 public class ConnectSFTP implements AutoCloseable {
@@ -9,33 +11,44 @@ public ConnectSFTP(){
     final String host = Sync.settings.get("ftp_host");
     final String port = Sync.settings.get("ftp_port");
     final String username = Sync.settings.get("ftp_username");
-    final String password = Sync.settings.get("ftp_password");
-    if (host==null || port==null || username==null || password==null){
+    final String key = Sync.settings.get("ftp_key");
+    if (host==null || port==null || username==null || key==null){
       Initializer.log("Failed to connect to SFTP server because connection settings were not provided.",true);
       return;
     }
     try{
-      connect(host, Integer.parseInt(port), username, password);
-    }catch(NumberFormatException e){
+      final ByteBuffer buf = ByteBuffer.wrap(key.getBytes(java.nio.charset.StandardCharsets.UTF_8));
+      synchronized(ConnectSFTP.class){
+        try(
+          FileChannel out = FileChannel.open(Initializer.sftpkey, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
+        ){
+          while (buf.hasRemaining()){
+            out.write(buf);
+          }
+        }
+      }
+      connect(host, Integer.parseInt(port), username);
+    }catch(Throwable e){
       Initializer.log(e);
       return;
     }
   }
-  public ConnectSFTP(String host, int port, String username, String password){
-    connect(host, port, username, password);
+  public ConnectSFTP(String host, int port, String username){
+    connect(host, port, username);
   }
-  private void connect(String host, int port, String username, String password){
+  private void connect(String host, int port, String username){
     try{
       final JSch jsch = new JSch();
       {
         final String knownHosts = Sync.settings.get("ftp_known_hosts");
         if (knownHosts==null){
+          Initializer.log("Failed to connect to SFTP server because ftp_known_hosts is undefined.",true);
           return;
         }
         jsch.setKnownHosts(new ByteArrayInputStream(knownHosts.getBytes(java.nio.charset.StandardCharsets.UTF_8)));
       }
+      jsch.addIdentity(Initializer.sftpkey.toString());
       jschSession = jsch.getSession(username, host, port);
-      jschSession.setPassword(password);
       jschSession.setTimeout(10000);
       jschSession.setConfig("StrictHostKeyChecking", "yes");
       jschSession.connect(10000);
@@ -47,6 +60,42 @@ private void connect(String host, int port, String username, String password){
       return;
     }
   }
+  public boolean overwriteFile(InputStream src, String dst){
+    if (jschChannel==null){
+      return false;
+    }
+    try{
+      jschChannel.put(src, dst, new SftpProgressMonitor(){
+        @Override public void init(int op, String src, String dest, long max){}
+        @Override public boolean count(long count){
+          return !Initializer.stop;
+        }
+        @Override public void end(){}
+      }, ChannelSftp.OVERWRITE);
+      return !Initializer.stop;
+    }catch(Throwable t){
+      Initializer.log(t);
+      return false;
+    }
+  }
+  public boolean uploadFile(Path src, String dst){
+    if (jschChannel==null){
+      return false;
+    }
+    try{
+      try(
+        BufferedInputStream s = new BufferedInputStream(Files.newInputStream(src));
+      ){
+        if (overwriteFile(s, dst)){
+          return true;
+        }
+      }
+      return false;
+    }catch(Throwable t){
+      Initializer.log(t);
+      return false;
+    }
+  }
   public boolean retrieveFile(String path, OutputStream out){
     if (jschChannel==null){
       return false;
@@ -76,7 +125,6 @@ public boolean downloadFile(String path, Path out){
         if (retrieveFile(path,s)){
           return true;
         }
-        s.close();
       }
       Files.deleteIfExists(out);
       return false;

src/aces/webctrl/postgresql/core/CronExpression.java

@@ -30,6 +30,12 @@ public boolean set(String expr){
       }
     }
   }
+  /**
+   * Explicity sets the next scheduled instant.
+   */
+  public void setNext(long next){
+    this.next = next;
+  }
   /**
    * Recomputes the next scheduled instant for this Cron expression.
    */
@@ -39,7 +45,12 @@ public void reset(){
       next = -1;
     }else{
       try{
-        next = gen.next(new Date()).getTime();
+        final long offset = Config.maxRandomOffset;
+        if (offset>0){
+          next = gen.next(new Date()).getTime()+(long)(offset*Math.random());
+        }else{
+          next = gen.next(new Date()).getTime();
+        }
       }catch(Throwable t){
         next = -1;
       }