diff --git a/.gitignore b/.gitignore index 4c49bd78f1d08f2bc09fa0bd8191ed38b7dce5e3..e4abf3723e57f1e064739ee4c02d3cf2212ecc5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .env +pentaho/kissflow/logs/* +pentaho/kissflow/kissflow_apps/* diff --git a/Dockerfile b/Dockerfile index 625d325f15c86c5b0955e5c84bd748d8e74c2c27..9ef3c8f78a6bea7942f7ee05c96316a569af012f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,11 @@ ENV PDI_RELEASE=7.1 \ KETTLE_HOME=/pentaho-di RUN apt-get update && apt-get install -y \ - libwebkitgtk-1.0.0 + libwebkitgtk-1.0.0 \ + python \ + python-pip + +RUN pip install requests RUN curl -L -o /tmp/pdi-ce-${PDI_VERSION}.zip \ https://superb-dca2.dl.sourceforge.net/project/pentaho/Data%20Integration/${PDI_RELEASE}/pdi-ce-${PDI_VERSION}.zip && \ @@ -16,10 +20,15 @@ RUN curl -L -o /tmp/pdi-ce-${PDI_VERSION}.zip \ ENV PATH=$PDI_HOME/data-integration:$PATH RUN mkdir -p $KETTLE_HOME/.kettle +RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ +COPY pentaho/ $KETTLE_HOME/jobs/ +RUN mkdir /var/log/kissflow +RUN mkdir -p $KETTLE_HOME/jobs/kissflow/kissflow_apps COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / +COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index 28286a9cc76403fbee59b63abb21768839f59bc9..9c0bb8a94320f15dc20ae7f06664a5e818d687f3 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,29 @@ CARTE_MASTER_PASSWORD ``` Variables for `kettle.properties` must be prefixed with `KP_`. + + +BUILD DOCKER: docker build -t pentaho . +RUN DOCKER: docker run -p 9081:9081 --env-file .env pentaho +CREATE A REQUEST: curl -X POST 'localhost:9081/kettle/executeJob/?job=/path/to/job/JOB_NAME.kjb&level=Debug' --user user:password + +## Kissflow + +Variables required for kissflow, with examples for nonsensitive data: +``` +KP_OUTPUT_DRIVER=org.postgresql.Driver +KP_OUTPUT_SCHEMA=public +KP_OUTPUT_PASSWORD +KP_OUTPUT_USERNAME +KP_OUTPUT_URL=jdbc\:postgresql\://HOSTNAME\:PORT/DATABASE_NAME +KP_KISSFLOW_SCRIPT_PATH=/pentaho-di/jobs/kissflow/run_kissflow.sh +KP_KISSFLOW_APPS=Lighting,CBRA,Sensors +KP_INPUT_DELIMITER=, +KP_INPUT_DIRECTORY=/pentaho-di/jobs/kissflow/kissflow_apps +KP_INPUT_ENCLOSURE=\" + +KISSFLOW_API_KEY +KISSFLOW_API_URL=https://kf-0003173.appspot.com/api/1/{}/list/p{}/{} +KISSFLOW_LOGGING_PATH=/var/log/kissflow/pylog-{} +KISSFLOW_CSV_PATH=/pentaho-di/jobs/kissflow/kissflow_apps/app_{}.csv +``` diff --git a/config/shared.xml b/config/shared.xml new file mode 100644 index 0000000000000000000000000000000000000000..8439711b85e6434f619b02918919607a036a5dc5 --- /dev/null +++ b/config/shared.xml @@ -0,0 +1,30 @@ + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + + + diff --git a/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr b/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr new file mode 100644 index 0000000000000000000000000000000000000000..f932bd18306963654674a219c14c1042c95e8994 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr @@ -0,0 +1,749 @@ + + + + DefineSqlVariables + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + + + + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/24 15:35:39.260 + - + 2016/10/25 12:50:14.859 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + ReadMetadataFile + DefineSqlVariablesWorker + Y + + + + DefineSqlVariablesWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/DefineSqlVariablesWorker.ktr + + ReadCsv + + + N + + + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + + + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + + + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + + + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + + + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + + + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + + + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + + + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + + + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + + + + + + + + + 288 + 64 + Y + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + + + + + + + 96 + 64 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr b/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr new file mode 100644 index 0000000000000000000000000000000000000000..bd37da9448eda13c479a473ba19e37388e8af487 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr @@ -0,0 +1,716 @@ + + + + DefineSqlVariablesWorker + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 12:43:03.614 + - + 2016/10/26 11:42:46.247 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + DefineSqlVariables + SetVariables + Y + + + ReadCsv + FirstRowOnly + Y + + + FirstRowOnly + DefineSqlVariables + Y + + + + DefineSqlVariables + UserDefinedJavaClass + + Y + + 1 + + none + + + + + TRANSFORM_CLASS + Processor + import org.pentaho.di.core.database.*; +import java.util.*; + +private boolean first = true; + +public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException +{ + Object[] row = this.getRow(); + + if (this.first) + { + this.first = false; + + String schemaName = this.getVariable("OUTPUT_SCHEMA", ""); + + + String tableName = this.getVariable("OUTPUT_TABLE", ""); + + RowMetaInterface inputRowMeta = this.getInputRowMeta(); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!1"); + + DatabaseMeta databaseMeta = this.getStepMeta().getParentTransMeta().findDatabase("Library_ExportCsvToTable_Connection"); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!2"); + logBasic(String.valueOf(databaseMeta)); + DatabaseMeta databaseMeta3 = this.getTransMeta().findDatabase("Library_ExportCsvToTable_Connection"); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!3"); + logBasic(String.valueOf(databaseMeta3)); + + Database database = new Database(this.getTrans(), databaseMeta3); + + String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName); + + String sqlCreate = database.getDDLCreationTable(schemaTable, inputRowMeta); + + String sqlDrop = "DROP TABLE " + schemaTable + ";"; + + Object[] outputRow = createOutputRow(row, this.data.outputRowMeta.size()); + + this.get(Fields.Out, "SqlCreate").setValue(outputRow, sqlCreate); + + this.get(Fields.Out, "SqlDrop").setValue(outputRow, sqlDrop); + + this.putRow(this.data.outputRowMeta, outputRow); + + return true; + } + + if (row != null) + { + return true; + } + else + { + this.setOutputDone(); + + return false; + } +} + + + + + SqlCreate + String + -1 + -1 + + + SqlDrop + String + -1 + -1 + + + Y + + + + + + + + + + + + 416 + 96 + Y + + + + FirstRowOnly + SampleRows + + Y + + 1 + + none + + + 1 + + + + + + + + + + 272 + 96 + Y + + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + + + + + + + 128 + 96 + Y + + + + SetVariables + SetVariable + + Y + + 1 + + none + + + + + SqlCreate + SQL_CREATE + PARENT_JOB + + + + SqlDrop + SQL_DROP + PARENT_JOB + + + + Y + + + + + + + + + 624 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb b/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb new file mode 100644 index 0000000000000000000000000000000000000000..103eb62ebf1d0ad06781d2940714a529d98766b4 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb @@ -0,0 +1,566 @@ + + + ExportCsvInDirectoryToDatabase + Exports all CSV files in a directory to homonymous tables in a database. + Exports all CSV files in a directory to homonymous tables in a database. + +Dependencies: +- /Library/ExportCsvToTable (folder) + + + 0 + / + - + 2016/10/25 13:40:04.860 + - + 2016/10/25 15:12:50.741 + + + INPUT_DELIMITER + , + The delimiter character in the CSV files in the input directory. + + + INPUT_DIRECTORY + + The input directory. + + + INPUT_ENCLOSURE + " + The enclosure character in the CSV files in the input directory. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output tables in. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 96 + 96 + + + GetCsvInDirectoryInformation + + TRANS + filename + + ${Internal.Entry.Current.Directory}/GetCsvInDirectoryInformation.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_DIRECTORY + + ${INPUT_DIRECTORY} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + N + Y + 0 + 256 + 96 + + + ExportCsvToTable + + JOB + filename + + ${Internal.Entry.Current.Directory}/ExportCsvToTable.kjb + + N + Y + Y + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + FilePath + + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + TableName + ${OUTPUT_TABLE} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + N + Y + 0 + 512 + 96 + + + Success + + SUCCESS + N + Y + 0 + 688 + 96 + + + + + START + GetCsvInDirectoryInformation + 0 + 0 + Y + Y + Y + + + ExportCsvToTable + Success + 0 + 0 + Y + Y + N + + + GetCsvInDirectoryInformation + ExportCsvToTable + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb new file mode 100644 index 0000000000000000000000000000000000000000..157e198d41e34a50e9b97d199fabf938051d77c1 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb @@ -0,0 +1,887 @@ + + + ExportCsvToTable + Exports a CSV file to a database table. + Exports a CSV file to a database table. + +Dependencies: +- /Library/ExtractCsvMetadataToCsv (folder) +- /Library_ExtractCsvMetadataToCsv_Connection (connection) + + 0 + / + - + 2016/10/23 21:53:12.195 + - + 2016/11/03 00:00:44.629 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output table in. + + + OUTPUT_TABLE + + The name of the output table. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + + CUSTOM_DRIVER_CLASS + ${OUTPUT_DRIVER} + + + CUSTOM_URL + ${OUTPUT_URL} + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + 1521 + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 64 + + + ExtractCsvMetadataToCsv + + JOB + filename + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsv.kjb + + N + N + N + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + OUTPUT_PATH + + ${METADATA_PATH} + + + OUTPUT_DELIMITER + + ${METADATA_DELIMITER} + + + OUTPUT_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + N + N + Y + 0 + 480 + 64 + + + DefineMetadataVariables + + EVAL + + N + Y + 0 + 256 + 64 + + + ExportCsvToTable + + TRANS + filename + + ${Internal.Entry.Current.Directory}/ExportCsvToTable.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + METADATA_DELIMITER + + ${METADATA_DELIMITER} + + + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + METADATA_PATH + + ${METADATA_PATH} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + + ${OUTPUT_TABLE} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + N + Y + 0 + 544 + 240 + + + DeleteMetadataFile + + DELETE_FILE + ${METADATA_PATH} + N + N + Y + 0 + 704 + 240 + + + DefineSqlVariables + + TRANS + filename + + ${Internal.Entry.Current.Directory}/DefineSqlVariables.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + METADATA_DELIMITER + + ${METADATA_DELIMITER} + + + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + METADATA_PATH + + ${METADATA_PATH} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + + ${OUTPUT_TABLE} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + Y + 0 + 720 + 64 + + + TableExists + + TABLE_EXISTS + ${OUTPUT_TABLE} + ${OUTPUT_SCHEMA} + Library_ExportCsvToTable_Connection + N + Y + 0 + 80 + 240 + + + DropTable + + SQL + ${SQL_DROP} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 240 + 304 + + + CreateTable + + SQL + ${SQL_CREATE} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 384 + 240 + + + Success + + SUCCESS + N + Y + 0 + 848 + 240 + + + + + ExportCsvToTable + DeleteMetadataFile + 0 + 0 + Y + Y + N + + + START + DefineMetadataVariables + 0 + 0 + Y + Y + Y + + + DefineMetadataVariables + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + N + + + ExtractCsvMetadataToCsv + DefineSqlVariables + 0 + 0 + Y + Y + N + + + DefineSqlVariables + TableExists + 0 + 0 + Y + Y + N + + + TableExists + DropTable + 0 + 0 + Y + Y + N + + + TableExists + CreateTable + 0 + 0 + Y + N + N + + + DropTable + CreateTable + 0 + 0 + Y + Y + N + + + CreateTable + ExportCsvToTable + 0 + 0 + Y + Y + N + + + DeleteMetadataFile + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr new file mode 100644 index 0000000000000000000000000000000000000000..d350115f5f365654dd2d87c48c2e6ff0794d63d6 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr @@ -0,0 +1,749 @@ + + + + ExportCsvToTable + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/23 18:35:54.307 + - + 2016/10/25 12:29:25.654 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + ReadMetadataFile + ExportCsvToTableWorker + Y + + + + ExportCsvToTableWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/ExportCsvToTableWorker.ktr + + ReadCsv + + + N + + + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + + + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + + + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + + + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + + + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + + + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + + + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + + + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + + + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + + + + + + + + + 352 + 96 + Y + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + + + + + + + 144 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvToTableWorker.ktr b/pentaho/kissflow/BlocPowerETL/ExportCsvToTableWorker.ktr new file mode 100644 index 0000000000000000000000000000000000000000..b3cdf45bce75e44584b00349c23244cbea9e5b8d --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvToTableWorker.ktr @@ -0,0 +1,229 @@ + + + ExportCsvToTableWorker + + + + Normal + 0 + /Library/ExportCsvToTable + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/23 16:55:31.957 + - + 2016/11/02 23:57:07.227 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + + + ReadCsvWriteTableY + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + 96 + 112 + Y + + + + + WriteTable + TableOutput + + Y + + 1 + + none + + + Library_ExportCsvToTable_Connection + ${OUTPUT_SCHEMA} +
${OUTPUT_TABLE}
+ 1000 + N + N + Y + N + N + + N + Y + N + + Y + N + + + + + + 256 + 112 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb new file mode 100644 index 0000000000000000000000000000000000000000..4eabedcab31c589d52a99a2b7a60e0e2285e5f6f --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb @@ -0,0 +1,460 @@ + + + ExtractCsvMetadataToCsv + Extracts metadata from a CSV file and saves it to another CSV file. + Extracts metadata from a CSV file and saves it to another CSV file. + +This job assumes that the input CSV file has at least two rows: +- The first is the header row (from which the names of the fields are taken). +- The second is a data row (from which the types of the fields are inferred). + + + 0 + / + - + 2016/10/22 16:22:05.789 + - + 2016/11/02 23:00:11.519 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DELIMITER + , + The delimiter character in the output metadata (CSV) file. + + + OUTPUT_ENCLOSURE + " + The delimiter character in the enclosure metadata (CSV) file. + + + OUTPUT_PATH + + The path of the new metadata (CSV) file. + + + + + + + + + + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 48 + + + ExtractCsvMetadataToCsv + + TRANS + filename + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsv.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + OUTPUT_DELIMITER + + ${OUTPUT_DELIMITER} + + + OUTPUT_ENCLOSURE + + ${OUTPUT_ENCLOSURE} + + + OUTPUT_PATH + + ${OUTPUT_PATH} + + + N + Y + 0 + 256 + 48 + + + Success + + SUCCESS + N + Y + 0 + 432 + 48 + + + + + START + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + Y + + + ExtractCsvMetadataToCsv + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr new file mode 100644 index 0000000000000000000000000000000000000000..be1b61832cf0453406d82251664ed11fdb829b93 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr @@ -0,0 +1,1116 @@ + + + + ExtractCsvMetadataToCsv + + + + Normal + 0 + / + + + INPUT_DELIMITER + , + + + + INPUT_ENCLOSURE + " + + + + INPUT_PATH + + + + + OUTPUT_DELIMITER + , + + + + OUTPUT_ENCLOSURE + " + + + + OUTPUT_PATH + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/22 13:07:22.373 + - + 2016/11/03 14:01:30.830 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + KeepHeaderOnly + BuildRowForEachField + Y + + + BuildRowForEachField + CleanRowSet + Y + + + CleanRowSet + PickFieldName + Y + + + PickFieldName + AddFieldTypeAsString + Y + + + AddFieldTypeAsString + ExtractCsvMetadataToCsvWorker + Y + + + ExtractCsvMetadataToCsvWorker + WriteCsv + Y + + + ReadLinesFromCsv + KeepHeaderOnly + Y + + + + AddFieldTypeAsString + Constant + + Y + + 1 + + none + + + + + FieldType + String + + + + + String + -1 + -1 + N + + + + + + + + + + + 240 + 224 + Y + + + + BuildRowForEachField + ScriptValueMod + + Y + + 1 + + none + + + N + 9 + + + 1 + ScriptStart + +// DEFAULT VALUES: + +var FieldName = null; + +// AUXILIARY FUNCTIONS: + +function SplitLine(line) +{ + var delimiter = getVariable("INPUT_DELIMITER",","); + var enclosure = getVariable("INPUT_ENCLOSURE", "\""); + var fields = []; + var currentField = ""; + var enclosureSeen = false; + + for (var i = 0; i < line.length; i++) + { + var character = line.charAt(i); + + if (enclosureSeen) + { + if (character == enclosure) + { + if (i < line.length - 1 && line.charAt(i + 1) == enclosure) + { + currentField += enclosure; + i += 1; + } + else + { + enclosureSeen = false; + } + } + else + { + currentField += character; + } + } + else + { + if (character == delimiter) + { + fields.push(currentField); + currentField = ""; + } + else if (character == enclosure) + { + enclosureSeen = true; + } + else + { + currentField += character; + } + } + } + + fields.push(currentField); + currentField = ""; + enclosureSeen = false; + + var regexp = new RegExp(enclosure, "g"); + + for (var i = 0; i < fields.length; i++) + { + fields[i] = fields[i].trim(); + + if (enclosure == "\"") + { + fields[i] = fields[i].replace(regexp, "'"); + } + } + + return fields; +} + + + 0 + ScriptLoop + +// LOOP CODE: + +var fields = SplitLine(LineText); + +for (var i = 0; i < fields.length; i++) +{ + var newRow = createRowCopy(getOutputRowMeta().size()); + + var rowIndex = getInputRowMeta().size(); + + // FieldName + newRow[rowIndex] = fields[i]; + + putRow(newRow); +} + + + + + + + + FieldName + FieldName + String + -1 + -1 + N + + + + + + + + + + + 416 + 64 + Y + + + + CleanRowSet + FilterRows + + Y + + 1 + + none + + + PickFieldName + + + + N + FieldName + IS NOT NULL + + + + + + + + + + + + 624 + 64 + Y + + + + ExtractCsvMetadataToCsvWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsvWorker.ktr + + PickMetadataFields + + + Name + String + -1 + -1 + + + Type + String + -1 + -1 + + + Format + String + -1 + -1 + + + Length + String + -1 + -1 + + + Precision + String + -1 + -1 + + + Currency + String + -1 + -1 + + + Decimal + String + -1 + -1 + + + Group + String + -1 + -1 + + + TrimType + String + -1 + -1 + + + + N + + + + + ReadCsv + FIELD_TYPE + Y + AddFieldTypeAsString + FieldType + + + ReadCsv + FIELD_NAME + Y + AddFieldTypeAsString + FieldName + + + + + + + + + + + 448 + 224 + Y + + + + KeepHeaderOnly + FilterRows + + Y + + 1 + + none + + + BuildRowForEachField + + + + N + LineNumber + = + + + constant + Integer + 1 + -1 + 0 + N + #;-# + + + + + + + + + + + + 240 + 64 + Y + + + + PickFieldName + SelectValues + + Y + + 1 + + none + + + + + FieldName + + N + + + + + + + + + + 80 + 224 + Y + + + + ReadLinesFromCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + LineNumber + N + $[00] + $[01] +
N
+ 50000 + Y + N + N + N + + + + LineText + String + + + + + -1 + -1 + none + + + + + + + + + + + 80 + 64 + Y + + + + WriteCsv + TextFileOutput + + Y + + 1 + + none + + + ${OUTPUT_DELIMITER} + ${OUTPUT_ENCLOSURE} + N + N +
Y
+
N
+ DOS + None + windows-1252 + + N + + Y + + ${OUTPUT_PATH} + N + N + N + + N + N + N + N + N + N + + Y + N + N + 0 + + + + Name + String + + + + + + none + -1 + -1 + + + Type + String + + + + + + none + -1 + -1 + + + Format + String + + + + + + none + -1 + -1 + + + Length + String + + + + + + none + -1 + -1 + + + Precision + String + + + + + + none + -1 + -1 + + + Currency + String + + + + + + none + -1 + -1 + + + Decimal + String + + + + + + none + -1 + -1 + + + Group + String + + + + + + none + -1 + -1 + + + TrimType + String + + + + + + none + -1 + -1 + + + + + + + + + + + 608 + 224 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsvWorker.ktr b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsvWorker.ktr new file mode 100644 index 0000000000000000000000000000000000000000..84af12831bf86c6cde5f5a9f615ce1874325ed2c --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsvWorker.ktr @@ -0,0 +1,462 @@ + + + ExtractCsvMetadataToCsvWorker + + + + Normal + 0 + /Library/ExtractCsvMetadataToCsv + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/22 14:58:15.112 + - + 2016/10/25 23:55:26.562 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + CleanRowSetPickMetadataFieldsY + ReadCsvDetectMetadataY + DetectMetadataCleanRowSetY + + + CleanRowSet + FilterRows + + Y + + 1 + + none + + +PickMetadataFields + + + + N + Name + IS NOT NULL + + + + + + 384 + 48 + Y + + + + + DetectMetadata + UserDefinedJavaClass + + Y + + 1 + + none + + + + + + TRANSFORM_CLASS + + Processor + + + + + + + Name + + String + + -1 + + -1 + + + + Type + + String + + -1 + + -1 + + + + Format + + String + + -1 + + -1 + + + + Length + + String + + -1 + + -1 + + + + Precision + + String + + -1 + + -1 + + + + Currency + + String + + -1 + + -1 + + + + Decimal + + String + + -1 + + -1 + + + + Group + + String + + -1 + + -1 + + + + TrimType + + String + + -1 + + -1 + + + Y + + + 240 + 48 + Y + + + + + PickMetadataFields + SelectValues + + Y + + 1 + + none + + + Name + + -2 + -2 + Type + + -2 + -2 + Format + + -2 + -2 + Length + + -2 + -2 + Precision + + -2 + -2 + Currency + + -2 + -2 + Decimal + + -2 + -2 + Group + + -2 + -2 + TrimType + + -2 + -2 + N + + + 576 + 48 + Y + + + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + 80 + 48 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr b/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr new file mode 100644 index 0000000000000000000000000000000000000000..3f3accb9a80d58fc7be3db225f19bec7b19da61d --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr @@ -0,0 +1,645 @@ + + + + GetCsvInDirectoryInformation + + + + Normal + 0 + / + + + INPUT_DIRECTORY + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 13:44:34.382 + - + 2016/10/25 14:47:13.667 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + GetCsvNames + SelectFields + Y + + + SelectFields + TableName + Y + + + TableName + SelectFields2 + Y + + + SelectFields2 + CopyRowsToResult + Y + + + + CopyRowsToResult + RowsToResult + + Y + + 1 + + none + + + + + + + + + + + 608 + 96 + Y + + + + GetCsvNames + GetFileNames + + Y + + 1 + + none + + + + only_files + + N + N + Y + N + + + + + N + 0 + + ${INPUT_DIRECTORY} + .*\.csv$ + + Y + N + + + + N + N + + + + + + + + + + 80 + 96 + Y + + + + SelectFields + SelectValues + + Y + + 1 + + none + + + + + filename + FilePath + + + short_filename + FileName + + N + + + + + + + + + + 208 + 96 + Y + + + + SelectFields2 + SelectValues + + Y + + 1 + + none + + + + + FilePath + + + TableName + + N + + + + + + + + + + 464 + 96 + Y + + + + TableName + Formula + + Y + + 1 + + none + + + + TableName + LEFT([FileName]; LEN([FileName])-4) + + String + -1 + -1 + + + + + + + + + + + 336 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb b/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb new file mode 100644 index 0000000000000000000000000000000000000000..4a80ec04b0f03839f20c0880ae03621d16d07570 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb @@ -0,0 +1,26 @@ + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + diff --git a/pentaho/kissflow/BlocPowerETL/Tester.kjb b/pentaho/kissflow/BlocPowerETL/Tester.kjb new file mode 100644 index 0000000000000000000000000000000000000000..63c73bfc545c7bd37865f21f327de90c0e78df0b --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Tester.kjb @@ -0,0 +1,419 @@ + + + Tester + Tester + Tester + +Dependencies: +- /Library (folder) + + 0 + / + - + 2016/10/25 15:13:28.377 + - + 2017/09/14 12:05:30.241 + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 144 + 96 + + + ExportCsvInDirectoryToDatabase + + JOB + filename + + ${Internal.Entry.Current.Directory}/ExportCsvInDirectoryToDatabase.kjb + + N + N + N + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_DIRECTORY + + ${INPUT_DIRECTORY} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + N + Y + 0 + 336 + 96 + + + Success + + SUCCESS + N + Y + 0 + 576 + 96 + + + + + START + ExportCsvInDirectoryToDatabase + 0 + 0 + Y + Y + Y + + + ExportCsvInDirectoryToDatabase + Success + 0 + 0 + Y + Y + N + + + + + diff --git a/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr new file mode 100644 index 0000000000000000000000000000000000000000..6777817a5f8f3c20dcf978e00441327751fea2d0 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr @@ -0,0 +1,629 @@ + + + + kissflow_script + + + + Normal + 0 + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2017/01/17 14:19:18.671 + - + 2017/09/15 12:00:55.538 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Get Variables + Split apps string into rows + Y + + + Create command + Execute a process + Y + + + Split apps string into rows + Create command + Y + + + + Create command + Calculator + + Y + + 1 + + none + + + + space + CONSTANT + + + + None + -1 + -1 + N + + + + + + + command + ADD3 + script_path + space + app_name + String + -1 + -1 + N + + + + + + + + + + + + + + 640 + 96 + Y + + + + Execute a process + ExecProcess + + Y + + 1 + + none + + + command + Result output + Error output + Exit value + Y + + N + + + + + + + + + + + 800 + 96 + Y + + + + Get Variables + GetVariable + + Y + + 1 + + none + + + + + apps_string + ${KISSFLOW_APPS} + String + + + + + -1 + -1 + none + + + script_path + ${KISSFLOW_SCRIPT_PATH} + String + + + + + -1 + -1 + none + + + + + + + + + + + 240 + 96 + Y + + + + Split apps string into rows + SplitFieldToRows3 + + Y + + 1 + + none + + + apps_string + , + app_name + N + + Y + N + + + + + + + + + 448 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/master_job.kjb b/pentaho/kissflow/BlocPowerETL/master_job.kjb new file mode 100644 index 0000000000000000000000000000000000000000..4b2c5f54516e12f3a3300988841a55b6ba1c45f3 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/master_job.kjb @@ -0,0 +1,438 @@ + + + master_job + + + + 0 + / + - + 2017/09/13 16:16:41.263 + - + 2017/09/15 18:07:52.592 + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 144 + 128 + + + Kissflow API to CSV + + TRANS + rep_name + + + kissflow_script + ${Internal.Entry.Current.Directory} + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 288 + 128 + + + CSV to DB + + JOB + filename + + ${Internal.Entry.Current.Directory}/Tester.kjb + + N + N + N + N + + log + N + N + Debug + + Y + N + N + N + N + + Y + + N + N + Y + 0 + 448 + 128 + + + Success + + SUCCESS + N + Y + 0 + 592 + 128 + + + + + START + Kissflow API to CSV + 0 + 0 + Y + Y + Y + + + Kissflow API to CSV + CSV to DB + 0 + 0 + Y + Y + N + + + CSV to DB + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py new file mode 100755 index 0000000000000000000000000000000000000000..a53ac2f70b0b092c95691b6ecd0e8b36a05586c8 --- /dev/null +++ b/pentaho/kissflow/kissflow.py @@ -0,0 +1,89 @@ +from datetime import datetime +import csv +import json +import requests +import sys +import time +import os + + +LOGGING_PATH = os.environ['KISSFLOW_LOGGING_PATH'] +CSV_PATH = os.environ['KISSFLOW_CSV_PATH'] +KISSFLOW_API_KEY = os.environ['KISSFLOW_API_KEY'] +KISSFLOW_API_URL = os.environ['KISSFLOW_API_URL'] +APP_NAME = sys.argv[1] +# If testing is true, only grab first 50 entries +TESTING = False +PAGE_SIZE = 50 +start = time.time() + +log_file = open(LOGGING_PATH.format(APP_NAME), 'w') +def log(s): + log_file.write('{}: {}\n'.format( + str(datetime.now()), + str(s), + )) + log_file.flush() + +log('Starting the script for app {}'.format(APP_NAME)) + +page_num = 1 +total_results = [] +while True: + response = requests.get(KISSFLOW_API_URL.format( + APP_NAME.replace('_', ' '), + page_num, + PAGE_SIZE, + ), headers={'api_key': KISSFLOW_API_KEY}) + if response.status_code != 200: + log('Error while making a request to {}'.format( + KISSFLOW_API_URL.format(APP_NAME, page_num, PAGE_SIZE) + )) + log('Status code was {}'.format(response.status_code)) + sys.exit(0) + json_result = response.json() + total_results.extend(json_result) + if len(json_result) < PAGE_SIZE: + break + + page_num += 1 + log('{} results so far...'.format(len(total_results))) + if TESTING: + break + +log('Get request succeeded with {} total requests retrieved'.format(len(total_results))) + +column_name_list = set() +for result in total_results: + for key in result: + column_name_list.add(key.encode('ascii', 'ignore')) +column_name_list = list(column_name_list) + +log('{} total unique column names'.format(len(column_name_list))) + +rowCounter = 1 +with open(CSV_PATH.format(APP_NAME), 'wb') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(column_name_list) + for result in total_results: + if rowCounter % 100 == 0: + log('Created {} rows so far..'.format(rowCounter)) + row = [] + for column in column_name_list: + if column in result: + field = result[column] + if type(field) is list: + field = json.dumps(field) + elif type(field) is not str: + try: + field = str(field) + except Exception as e: + field = str(field.encode('ascii', 'ignore')) + + row.append(field.encode('ascii', 'ignore')) + else: + row.append(None) + csvwriter.writerow(row) + rowCounter += 1 +end = time.time() +log('Completed script in {} seconds'.format(end - start)) diff --git a/pentaho/kissflow/run_kissflow.sh b/pentaho/kissflow/run_kissflow.sh new file mode 100755 index 0000000000000000000000000000000000000000..93c67155e5d43f143b9d709b08a24cbcd8f20ef9 --- /dev/null +++ b/pentaho/kissflow/run_kissflow.sh @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/python /pentaho-di/jobs/kissflow/kissflow.py $1