#789 - Created new tables;#801 - Closed connections; #796 - Created temp files properly

Author: vivekmuniyandi

src/main/java/com/marklogic/client/example/cookbook/datamovement/BulkExportOpticResults.java

@@ -52,7 +52,6 @@ public class BulkExportOpticResults {
   private int batchSize = 3;
   private int threadCount = 3;
   private String schemaDB = "Schemas";
-  private String outputFile = "/tmp/opticExampleOutput.txt";
   private String customerTemplateFile = "/opticExample/customer.tdex";
   private String orderTemplateFile = "/opticExample/order.tdex";
   private DatabaseClient schemaDBclient = DatabaseClientSingleton.getAdmin(schemaDB);
@@ -192,7 +191,6 @@ private void exportWithOptic(int productID) throws IOException {
     // Create a Row manager to construct plans and query on
     // rows projected from the documents
     RowManager rowMgr = client.newRowManager();
-    try (FileWriter writer = new FileWriter(outputFile)) {
       // Create a QueryBatcher to get the uris from the database which match the
       // query definition and pass it to the OpticExportToWriterListener for
       // creating the Optic API plans and writing the results of the plan to the
@@ -237,7 +235,6 @@ private void exportWithOptic(int productID) throws IOException {
       // Wait till the batch completes
       queryBatcher.awaitCompletion();
       moveMgr.stopJob(queryBatcher);
-    }
   }
 
   private void tearDown() {

src/main/java/com/marklogic/client/example/cookbook/datamovement/BulkExportOpticResultsToWriter.java

@@ -59,7 +59,7 @@ public class BulkExportOpticResultsToWriter {
   private int batchSize = 3;
   private int threadCount = 3;
   private String schemaDB = "Schemas";
-  private String outputFile = "/tmp/opticExampleOutput.txt";
+  private String outputFile = System.getProperty("java.io.tmpdir") + "opticExampleOutput.txt";
   private String customerTemplateFile = "/opticExample/customer.tdex";
   private String orderTemplateFile = "/opticExample/order.tdex";
   private DatabaseClient schemaDBclient = DatabaseClientSingleton.getAdmin(schemaDB);
@@ -199,6 +199,7 @@ private void exportWithOptic(int productID) throws IOException {
     // Create a Row manager to construct plans and query on
     // rows projected from the documents
     RowManager rowMgr = client.newRowManager();
+    System.out.println("Writing the results to " + outputFile);
     try (FileWriter writer = new FileWriter(outputFile)) {
       // Create a Function to pass to the OpticExportToWriterListener which
       // would take each batch and do the necessary optic operations and return

src/main/java/com/marklogic/client/example/cookbook/datamovement/BulkExportToJdbc.java

@@ -43,6 +43,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Calendar;
@@ -70,12 +71,6 @@ public void run() throws IOException, SQLException {
     JdbcTemplate jdbcTemplate = new JdbcTemplate(getDataSource());
     final boolean isMySQLDB;
     SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
-    if (jdbcTemplate.getDataSource().getConnection().getMetaData().getDatabaseProductName().toLowerCase()
-        .contains("hsql")) {
-      isMySQLDB = false;
-    } else {
-      isMySQLDB = true;
-    }
     StructuredQueryDefinition query = new StructuredQueryBuilder().directory(true, "/employees/");
     QueryBatcher qb = moveMgr.newQueryBatcher(query)
       .withBatchSize(batchSize)
@@ -85,72 +80,28 @@ public void run() throws IOException, SQLException {
           .withConsistentSnapshot()
           .onDocumentReady(record -> {
             Employee employee = record.getContentAs(Employee.class);
-            if(isMySQLDB) {
-              jdbcTemplate.update(
-                "INSERT INTO employees (emp_no, hire_date, first_name, last_name, birth_date) " +
-                "VALUES (?, ?, ?, ?, ?) " +
-                "ON DUPLICATE KEY UPDATE emp_no=?, hire_date=?, first_name=?, last_name=?, birth_date=?",
-                employee.getEmployeeId(), employee.getHireDate(), employee.getFirstName(),
-                employee.getLastName(), employee.getBirthDate(),
-                employee.getEmployeeId(), employee.getHireDate(), employee.getFirstName(),
-                employee.getLastName(), employee.getBirthDate());
-            } else {
-              jdbcTemplate.update(
-                "MERGE INTO employees USING (VALUES ?, ?, ?, ?, ?, ?)  " +
-                    "temp (emp_no, hire_date, first_name, last_name, gender, birth_date) " +
-                "ON employees.emp_no = temp.emp_no " +
-                "WHEN MATCHED THEN UPDATE SET employees.hire_date=temp.hire_date, " +
-                  "employees.first_name=temp.first_name, employees.last_name=temp.last_name, " +
-                  "employees.gender = temp.gender, employees.birth_date=temp.birth_date " +
-                "WHEN NOT MATCHED THEN INSERT (emp_no, hire_date, first_name, last_name, gender, birth_date)  "
-                + "VALUES (temp.emp_no, temp.hire_date, temp.first_name, temp.last_name, temp.gender, temp.birth_date) ",
-                employee.getEmployeeId(), dateFormat.format(employee.getHireDate().getTime()), employee.getFirstName(),
-                employee.getLastName(), employee.getGender() == Gender.MALE ? "M" : "F",
-                dateFormat.format(employee.getBirthDate().getTime()));
-            }
+            jdbcTemplate.update(
+              "INSERT INTO employees_export (emp_no, hire_date, first_name, last_name, gender, birth_date) " +
+              "VALUES (?, ?, ?, ?, ?, ?) ",
+              employee.getEmployeeId(), dateFormat.format(employee.getHireDate().getTime()), employee.getFirstName(),
+              employee.getLastName(), employee.getGender() == Gender.MALE ? "M" : "F",
+              dateFormat.format(employee.getBirthDate().getTime()));
             if ( employee.getSalaries() != null ) {
               for ( Salary salary : employee.getSalaries() ) {
-                if(isMySQLDB) {
-                  jdbcTemplate.update(
-                    "INSERT INTO salaries (emp_no, salary, from_date, to_date) " +
-                    "VALUES(?, ?, ?, ?)" +
-                    "ON DUPLICATE KEY UPDATE emp_no=?, salary=?, from_date=?, to_date=?",
-                    employee.getEmployeeId(), salary.getSalary(), salary.getFromDate(), salary.getToDate(),
-                    employee.getEmployeeId(), salary.getSalary(), salary.getFromDate(), salary.getToDate());
-                } else {
-                  jdbcTemplate.update(
-                    "MERGE INTO salaries USING (VALUES ?, ?, ?, ?) " +
-                        "temp (emp_no, salary, from_date, to_date) " +
-                    "ON salaries.emp_no = temp.emp_no AND salaries.from_date = temp.from_date " +
-                    "WHEN MATCHED THEN UPDATE SET salaries.salary=temp.salary, " +
-                      "salaries.to_date=temp.to_date " +
-                    "WHEN NOT MATCHED THEN INSERT (emp_no, salary, from_date, to_date) " +
-                    "VALUES (temp.emp_no, temp.salary, temp.from_date, temp.to_date)",
-                    employee.getEmployeeId(), salary.getSalary(), dateFormat.format(salary.getFromDate().getTime()),
-                    dateFormat.format(salary.getToDate().getTime()));
-                }
+                jdbcTemplate.update(
+                  "INSERT INTO salaries_export (emp_no, salary, from_date, to_date) " +
+                  "VALUES(?, ?, ?, ?)",
+                  employee.getEmployeeId(), salary.getSalary(), dateFormat.format(salary.getFromDate().getTime()),
+                  dateFormat.format(salary.getToDate().getTime()));
               }
             }
             if ( employee.getTitles() != null ) {
               for ( Title title : employee.getTitles() ) {
-                if(isMySQLDB) {
-                  jdbcTemplate.update(
-                    "INSERT INTO titles (emp_no, title, from_date, to_date) " +
-                    "VALUES(?, ?, ?, ?)" +
-                    "ON DUPLICATE KEY UPDATE emp_no=?, title=?, from_date=?, to_date=?",
-                    employee.getEmployeeId(), title.getTitle(), title.getFromDate(), title.getToDate(),
-                    employee.getEmployeeId(), title.getTitle(), title.getFromDate(), title.getToDate());
-                } else {
-                  jdbcTemplate.update(
-                    "MERGE INTO titles USING (VALUES ?, ?, ?, ?)  " +
-                        "temp (emp_no, title, from_date, to_date) " +
-                    "ON titles.emp_no = temp.emp_no AND titles.title = temp.title AND titles.from_date=temp.from_date " +
-                    "WHEN MATCHED THEN UPDATE SET titles.to_date=temp.to_date " +
-                    "WHEN NOT MATCHED THEN INSERT (emp_no, title, from_date, to_date) " +
-                    "VALUES (temp.emp_no, temp.title, temp.from_date, temp.to_date)",
-                    employee.getEmployeeId(), title.getTitle(), dateFormat.format(title.getFromDate().getTime()),
-                    dateFormat.format(title.getToDate().getTime()));
-                }
+                jdbcTemplate.update(
+                  "INSERT INTO titles_export (emp_no, title, from_date, to_date) " +
+                  "VALUES(?, ?, ?, ?)",
+                  employee.getEmployeeId(), title.getTitle(), dateFormat.format(title.getFromDate().getTime()),
+                  dateFormat.format(title.getToDate().getTime()));
               }
             }
           })

src/main/java/com/marklogic/client/example/cookbook/datamovement/BulkLoadFromJdbcWithJoins.java

@@ -35,6 +35,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Calendar;
@@ -69,9 +70,11 @@ public Employee parseEmployee(ResultSet row) throws SQLException {
 
   public void run() throws IOException, SQLException {
     JdbcTemplate jdbcTemplate = new JdbcTemplate(getDataSource());
-    if(jdbcTemplate.getDataSource().getConnection().getMetaData().getDatabaseProductName().toLowerCase().contains("mysql"))
+    Connection conn = jdbcTemplate.getDataSource().getConnection();
+    if(conn.getMetaData().getDatabaseProductName().toLowerCase().contains("mysql"))
       // the following is required because GROUP_CONCAT calls in following SQL can generate long strings
       jdbcTemplate.execute("SET GLOBAL group_concat_max_len = 1000000");
+    conn.close();
     WriteBatcher wb = moveMgr.newWriteBatcher()
       .withBatchSize(batchSize)
       .withThreadCount(threadCount)

src/test/java/com/marklogic/client/test/example/cookbook/DMSDKJdbcCookbookTests.java

@@ -92,6 +92,31 @@ private void populateDataset(Database hsqlDB) {
     "FOREIGN KEY (emp_no) REFERENCES employees (emp_no) ON DELETE CASCADE," +
     "PRIMARY KEY (emp_no, from_date));"); 
 
+    hsqlDB.execute("CREATE TABLE employees_export (" +
+        "emp_no      INTEGER         NOT NULL," +
+        "birth_date  DATE            NOT NULL," +
+        "first_name  VARCHAR(14)     NOT NULL," +
+        "last_name   VARCHAR(16)     NOT NULL," +
+        "gender      VARCHAR(2)      NOT NULL," +
+        "hire_date   DATE            NOT NULL," +
+        "PRIMARY KEY (emp_no));");
+
+      hsqlDB.execute("CREATE TABLE titles_export (" +
+        "emp_no      INTEGER         NOT NULL," +
+        "title       VARCHAR(50)     NOT NULL," +
+        "from_date   DATE            NOT NULL," +
+        "to_date     DATE," +
+        "FOREIGN KEY (emp_no) REFERENCES employees_export (emp_no) ON DELETE CASCADE," +
+        "PRIMARY KEY (emp_no,title, from_date));");
+
+      hsqlDB.execute("CREATE TABLE salaries_export (" +
+      "emp_no      INTEGER         NOT NULL," +
+      "salary      INTEGER         NOT NULL," +
+      "from_date   DATE            NOT NULL," +
+      "to_date     DATE            NOT NULL," +
+      "FOREIGN KEY (emp_no) REFERENCES employees_export (emp_no) ON DELETE CASCADE," +
+      "PRIMARY KEY (emp_no, from_date));");
+
     hsqlDB.execute("INSERT INTO employees VALUES (1, '1990-10-04', 'Alice', 'Edward', 'F', '2012-04-05');");
     hsqlDB.execute("INSERT INTO employees VALUES (2, '1992-12-23', 'Bob', 'Miller', 'M', '2010-06-01');");
     hsqlDB.execute("INSERT INTO employees VALUES (3, '1985-11-30', 'Gerard', 'Steven', 'M', '2011-07-29');");