diff --git a/embulk-output-jdbc/README.md b/embulk-output-jdbc/README.md index ca355e1f..e986f4fd 100644 --- a/embulk-output-jdbc/README.md +++ b/embulk-output-jdbc/README.md @@ -25,6 +25,7 @@ Generic JDBC output plugin for Embulk loads records to a database using a JDBC d - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second)) - **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes)) - **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See below (string, required) +- **single_intermediate_table**: Regardless of the mode, at most one intermediate table will be created. This is primarily used to avoid creating a large number of intermediate tables when there are a large number of input files. (boolean, default: false) - **batch_size**: size of a single batch insert (integer, default: 16777216) - **max_table_name_length**: maximum length of table name in this RDBMS (integer, default: 256) - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`) diff --git a/embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java b/embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java index 9f2c1cb9..15b623b6 100644 --- a/embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +++ b/embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java @@ -94,6 +94,10 @@ public interface PluginTask @Config("mode") public Mode getMode(); + @Config("single_intermediate_table") + @ConfigDefault("false") + public boolean getSingleIntermediateTable(); + @Config("batch_size") @ConfigDefault("16777216") // TODO set minimum number @@ -693,7 +697,7 @@ private List createIntermediateTables(final JdbcOutputConnectio public List call() throws Exception { intermTables = new ArrayList<>(); - if (task.getMode().tempTablePerTask()) { + if (task.getMode().tempTablePerTask() && !task.getSingleIntermediateTable()) { String tableNameFormat = generateIntermediateTableNameFormat(task.getActualTable().getTableName(), con, taskCount, task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics()); for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) { @@ -1078,7 +1082,7 @@ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final // configure BatchInsert -> an intermediate table (!isDirectModify) or the target table (isDirectModify) TableIdentifier destTable; - if (mode.tempTablePerTask()) { + if (mode.tempTablePerTask() && !task.getSingleIntermediateTable()) { destTable = task.getIntermediateTables().get().get(taskIndex); } else if (mode.isDirectModify()) { destTable = task.getActualTable();