From 72b3036c46e93efeb87e62993aa752a774f264a9 Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 Sep 2017 12:07:58 -0400 Subject: [PATCH 01/11] Add kissflow pentaho job --- .gitignore | 2 + .../ExportCsvInDirectoryToDatabase.kjb | 256 +++++++ .../GetCsvInDirectoryInformation.ktr | 239 +++++++ .../ExportCsvToTable/DefineSqlVariables.ktr | 353 ++++++++++ .../DefineSqlVariablesWorker.ktr | 323 +++++++++ .../ExportCsvToTable/ExportCsvToTable.kjb | 491 ++++++++++++++ .../ExportCsvToTable/ExportCsvToTable.ktr | 353 ++++++++++ .../ExportCsvToTableWorker.ktr | 229 +++++++ .../ExtractCsvMetadataToCsv.kjb | 172 +++++ .../ExtractCsvMetadataToCsv.ktr | 550 +++++++++++++++ .../ExtractCsvMetadataToCsvWorker.ktr | 462 +++++++++++++ .../Library_ExportCsvToTable_Connection.kdb | 26 + .../BlocPowerETL/Programs/Tester/Tester.kjb | 419 ++++++++++++ .../kissflow/BlocPowerETL/kissflow_script.ktr | 628 ++++++++++++++++++ pentaho/kissflow/BlocPowerETL/master_job.kjb | 438 ++++++++++++ pentaho/kissflow/BlocPowerETL/repository.log | 10 + pentaho/kissflow/ddl/create_kissflow_db..sql | 406 +++++++++++ pentaho/kissflow/kissflow.py | 84 +++ pentaho/kissflow/kissflow_transformation.ktr | 526 +++++++++++++++ pentaho/kissflow/run_kissflow.sh | 2 + .../scripts/RestAPI data Retrieve Script | 32 + .../kissflow/scripts/RestAPI update Script.sh | 29 + 22 files changed, 6030 insertions(+) create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb create mode 100644 pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/kissflow_script.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/master_job.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/repository.log create mode 100644 pentaho/kissflow/ddl/create_kissflow_db..sql create mode 100644 pentaho/kissflow/kissflow.py create mode 100644 pentaho/kissflow/kissflow_transformation.ktr create mode 100755 pentaho/kissflow/run_kissflow.sh create mode 100644 pentaho/kissflow/scripts/RestAPI data Retrieve Script create mode 100644 pentaho/kissflow/scripts/RestAPI update Script.sh diff --git a/.gitignore b/.gitignore index 4c49bd7..e4abf37 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .env +pentaho/kissflow/logs/* +pentaho/kissflow/kissflow_apps/* diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb new file mode 100644 index 0000000..e8e3af8 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb @@ -0,0 +1,256 @@ + + ExportCsvInDirectoryToDatabase + Exports all CSV files in a directory to homonymous tables in a database. + Exports all CSV files in a directory to homonymous tables in a database. Dependencies: - /Library/ExportCsvToTable (folder) + + 0 + /Library/ExportCsvInDirectoryToDatabase + - + 2016/10/25 13:40:04.860 + - + 2016/10/25 15:12:50.741 + + + INPUT_DELIMITER + , + The delimiter character in the CSV files in the input directory. + + + INPUT_DIRECTORY + + The input directory. + + + INPUT_ENCLOSURE + " + The enclosure character in the CSV files in the input directory. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output tables in. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + + + + + + + +ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 96 + 96 + + + GetCsvInDirectoryInformation + + TRANS + rep_name + + + GetCsvInDirectoryInformation + /Library/ExportCsvInDirectoryToDatabase + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_DIRECTORY + + ${INPUT_DIRECTORY} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + OUTPUT_URL + + ${OUTPUT_URL} + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + N + Y + 0 + 240 + 96 + + + ExportCsvToTable + + JOB + rep_name + + + ExportCsvToTable + /Library/ExportCsvToTable + N + Y + Y + N + + + N + N + Nothing + + Y + N + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + INPUT_PATH + FilePath + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + OUTPUT_TABLE + TableName + ${OUTPUT_TABLE} + OUTPUT_URL + + ${OUTPUT_URL} + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + N + N + Y + 0 + 416 + 96 + + + Success + + SUCCESS + N + Y + 0 + 576 + 96 + + + + + START + GetCsvInDirectoryInformation + 0 + 0 + Y + Y + Y + + + GetCsvInDirectoryInformation + ExportCsvToTable + 0 + 0 + Y + Y + N + + + ExportCsvToTable + Success + 0 + 0 + Y + Y + N + + + + + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr new file mode 100644 index 0000000..e0bef1e --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr @@ -0,0 +1,239 @@ + + + GetCsvInDirectoryInformation + + + + Normal + 0 + /Library/ExportCsvInDirectoryToDatabase + + + INPUT_DIRECTORY + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 13:44:34.382 + - + 2016/10/25 14:47:13.667 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + GetCsvNamesSelectFieldsY + SelectFieldsTableNameY + TableNameSelectFields2Y + SelectFields2CopyRowsToResultY + + + CopyRowsToResult + RowsToResult + + Y + + 1 + + none + + + + + 608 + 96 + Y + + + + + GetCsvNames + GetFileNames + + Y + + 1 + + none + + + + only_files + + N + N + Y + N + + + + + N + 0 + + ${INPUT_DIRECTORY} + .*\.csv$ + + N + N + + + + N + N + + + + 80 + 96 + Y + + + + + SelectFields + SelectValues + + Y + + 1 + + none + + + filename + FilePath + -2 + -2 + short_filename + FileName + -2 + -2 + N + + + 208 + 96 + Y + + + + + SelectFields2 + SelectValues + + Y + + 1 + + none + + + FilePath + + -2 + -2 + TableName + + -2 + -2 + N + + + 464 + 96 + Y + + + + + TableName + Formula + + Y + + 1 + + none + + + TableName +LEFT([FileName]; LEN([FileName])-4) +String +-1 +-1 + + + + + 336 + 96 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr new file mode 100644 index 0000000..eba0c55 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr @@ -0,0 +1,353 @@ + + + DefineSqlVariables + + + + Normal + 0 + /Library/ExportCsvToTable + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/24 15:35:39.260 + - + 2016/10/25 12:50:14.859 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + ReadMetadataFileDefineSqlVariablesWorkerY + + + DefineSqlVariablesWorker + MetaInject + + Y + + 1 + + none + + + rep_name + + DefineSqlVariablesWorker + + /Library/ExportCsvToTable + ReadCsv + + N + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + 288 + 64 + Y + + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + 112 + 64 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr new file mode 100644 index 0000000..9de0340 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr @@ -0,0 +1,323 @@ + + + DefineSqlVariablesWorker + + + + Normal + 0 + /Library/ExportCsvToTable + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 12:43:03.614 + - + 2016/10/26 11:42:46.247 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + DefineSqlVariablesSetVariablesY + ReadCsvFirstRowOnlyY + FirstRowOnlyDefineSqlVariablesY + + + DefineSqlVariables + UserDefinedJavaClass + + Y + + 1 + + none + + + + + + TRANSFORM_CLASS + + Processor + + + + + + + SqlCreate + + String + + -1 + + -1 + + + + SqlDrop + + String + + -1 + + -1 + + + Y + + + 432 + 96 + Y + + + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + 128 + 96 + Y + + + + + SetVariables + SetVariable + + Y + + 1 + + none + + + + + SqlCreate + SQL_CREATE + PARENT_JOB + + + + SqlDrop + SQL_DROP + PARENT_JOB + + + + Y + + + 624 + 96 + Y + + + + + FirstRowOnly + SampleRows + + Y + + 1 + + none + + + 1 + + + + 272 + 96 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb new file mode 100644 index 0000000..a3d577a --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb @@ -0,0 +1,491 @@ + + ExportCsvToTable + Exports a CSV file to a database table. + Exports a CSV file to a database table. Dependencies: - /Library/ExtractCsvMetadataToCsv (folder) - /Library_ExtractCsvMetadataToCsv_Connection (connection) + + 0 + /Library/ExportCsvToTable + - + 2016/10/23 21:53:12.195 + - + 2016/11/03 00:00:44.629 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output table in. + + + OUTPUT_TABLE + + The name of the output table. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSY + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + + + + + +
+ + + +ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 64 + + + ExtractCsvMetadataToCsv + + JOB + rep_name + + + ExtractCsvMetadataToCsv + /Library/ExtractCsvMetadataToCsv + N + N + N + N + + + N + N + Nothing + + Y + N + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + INPUT_PATH + + ${INPUT_PATH} + OUTPUT_PATH + + ${METADATA_PATH} + OUTPUT_DELIMITER + + ${METADATA_DELIMITER} + OUTPUT_ENCLOSURE + + ${METADATA_ENCLOSURE} + N + N + Y + 0 + 480 + 64 + + + DefineMetadataVariables + + EVAL + + N + Y + 0 + 256 + 64 + + + ExportCsvToTable + + TRANS + rep_name + + + ExportCsvToTable + /Library/ExportCsvToTable + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + INPUT_PATH + + ${INPUT_PATH} + METADATA_DELIMITER + + ${METADATA_DELIMITER} + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + METADATA_PATH + + ${METADATA_PATH} + OUTPUT_URL + + ${OUTPUT_URL} + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + OUTPUT_TABLE + + ${OUTPUT_TABLE} + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + N + Y + 0 + 544 + 240 + + + DeleteMetadataFile + + DELETE_FILE + ${METADATA_PATH} + N + N + Y + 0 + 704 + 240 + + + DefineSqlVariables + + TRANS + rep_name + + + DefineSqlVariables + /Library/ExportCsvToTable + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + INPUT_PATH + + ${INPUT_PATH} + METADATA_DELIMITER + + ${METADATA_DELIMITER} + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + METADATA_PATH + + ${METADATA_PATH} + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + OUTPUT_TABLE + + ${OUTPUT_TABLE} + OUTPUT_URL + + ${OUTPUT_URL} + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + N + Y + 0 + 720 + 64 + + + TableExists + + TABLE_EXISTS + ${OUTPUT_TABLE} + ${OUTPUT_SCHEMA} + Library_ExportCsvToTable_Connection + N + Y + 0 + 80 + 240 + + + DropTable + + SQL + ${SQL_DROP} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 240 + 304 + + + CreateTable + + SQL + ${SQL_CREATE} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 384 + 240 + + + Success + + SUCCESS + N + Y + 0 + 848 + 240 + + + + + ExportCsvToTable + DeleteMetadataFile + 0 + 0 + Y + Y + N + + + START + DefineMetadataVariables + 0 + 0 + Y + Y + Y + + + DefineMetadataVariables + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + N + + + ExtractCsvMetadataToCsv + DefineSqlVariables + 0 + 0 + Y + Y + N + + + DefineSqlVariables + TableExists + 0 + 0 + Y + Y + N + + + TableExists + DropTable + 0 + 0 + Y + Y + N + + + TableExists + CreateTable + 0 + 0 + Y + N + N + + + DropTable + CreateTable + 0 + 0 + Y + Y + N + + + CreateTable + ExportCsvToTable + 0 + 0 + Y + Y + N + + + DeleteMetadataFile + Success + 0 + 0 + Y + Y + N + + + + + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr new file mode 100644 index 0000000..ebdd0ca --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr @@ -0,0 +1,353 @@ + + + ExportCsvToTable + + + + Normal + 0 + /Library/ExportCsvToTable + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/23 18:35:54.307 + - + 2016/10/25 12:29:25.654 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + ReadMetadataFileExportCsvToTableWorkerY + + + ExportCsvToTableWorker + MetaInject + + Y + + 1 + + none + + + rep_name + + ExportCsvToTableWorker + + /Library/ExportCsvToTable + ReadCsv + + N + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + 320 + 96 + Y + + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + 144 + 96 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr new file mode 100644 index 0000000..b3cdf45 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr @@ -0,0 +1,229 @@ + + + ExportCsvToTableWorker + + + + Normal + 0 + /Library/ExportCsvToTable + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/23 16:55:31.957 + - + 2016/11/02 23:57:07.227 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + + + ReadCsvWriteTableY + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + 96 + 112 + Y + + + + + WriteTable + TableOutput + + Y + + 1 + + none + + + Library_ExportCsvToTable_Connection + ${OUTPUT_SCHEMA} +
${OUTPUT_TABLE}
+ 1000 + N + N + Y + N + N + + N + Y + N + + Y + N + + + + + + 256 + 112 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb new file mode 100644 index 0000000..cb33798 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb @@ -0,0 +1,172 @@ + + ExtractCsvMetadataToCsv + Extracts metadata from a CSV file and saves it to another CSV file. + Extracts metadata from a CSV file and saves it to another CSV file. This job assumes that the input CSV file has at least two rows: - The first is the header row (from which the names of the fields are taken). - The second is a data row (from which the types of the fields are inferred). + + 0 + /Library/ExtractCsvMetadataToCsv + - + 2016/10/22 16:22:05.789 + - + 2016/11/02 23:00:11.519 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DELIMITER + , + The delimiter character in the output metadata (CSV) file. + + + OUTPUT_ENCLOSURE + " + The delimiter character in the enclosure metadata (CSV) file. + + + OUTPUT_PATH + + The path of the new metadata (CSV) file. + + + + + + + + + + +ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 48 + + + ExtractCsvMetadataToCsv + + TRANS + rep_name + + + ExtractCsvMetadataToCsv + /Library/ExtractCsvMetadataToCsv + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + N + INPUT_DELIMITER + + ${INPUT_DELIMITER} + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + INPUT_PATH + + ${INPUT_PATH} + OUTPUT_DELIMITER + + ${OUTPUT_DELIMITER} + OUTPUT_ENCLOSURE + + ${OUTPUT_ENCLOSURE} + OUTPUT_PATH + + ${OUTPUT_PATH} + N + Y + 0 + 256 + 48 + + + Success + + SUCCESS + N + Y + 0 + 432 + 48 + + + + + START + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + Y + + + ExtractCsvMetadataToCsv + Success + 0 + 0 + Y + Y + N + + + + + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr new file mode 100644 index 0000000..8ca208c --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr @@ -0,0 +1,550 @@ + + + ExtractCsvMetadataToCsv + + + + Normal + 0 + /Library/ExtractCsvMetadataToCsv + + + INPUT_DELIMITER + , + + + + INPUT_ENCLOSURE + " + + + + INPUT_PATH + + + + + OUTPUT_DELIMITER + , + + + + OUTPUT_ENCLOSURE + " + + + + OUTPUT_PATH + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/22 13:07:22.373 + - + 2016/11/03 14:01:30.830 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + KeepHeaderOnlyBuildRowForEachFieldY + BuildRowForEachFieldCleanRowSetY + CleanRowSetPickFieldNameY + PickFieldNameAddFieldTypeAsStringY + AddFieldTypeAsStringExtractCsvMetadataToCsvWorkerY + ExtractCsvMetadataToCsvWorkerWriteCsvY + ReadLinesFromCsvKeepHeaderOnlyY + + + AddFieldTypeAsString + Constant + + Y + + 1 + + none + + + + + FieldType + String + + + + + String + -1 + -1 + N + + + + + 240 + 224 + Y + + + + + BuildRowForEachField + ScriptValueMod + + Y + + 1 + + none + + + N + 9 + 1 + ScriptStart + // DEFAULT VALUES: var FieldName = null; // AUXILIARY FUNCTIONS: function SplitLine(line) { var delimiter = getVariable("INPUT_DELIMITER",","); var enclosure = getVariable("INPUT_ENCLOSURE", "\""); var fields = []; var currentField = ""; var enclosureSeen = false; for (var i = 0; i < line.length; i++) { var character = line.charAt(i); if (enclosureSeen) { if (character == enclosure) { if (i < line.length - 1 && line.charAt(i + 1) == enclosure) { currentField += enclosure; i += 1; } else { enclosureSeen = false; } } else { currentField += character; } } else { if (character == delimiter) { fields.push(currentField); currentField = ""; } else if (character == enclosure) { enclosureSeen = true; } else { currentField += character; } } } fields.push(currentField); currentField = ""; enclosureSeen = false; var regexp = new RegExp(enclosure, "g"); for (var i = 0; i < fields.length; i++) { fields[i] = fields[i].trim(); if (enclosure == "\"") { fields[i] = fields[i].replace(regexp, "'"); } } return fields; } + 0 + ScriptLoop + // LOOP CODE: var fields = SplitLine(LineText); for (var i = 0; i < fields.length; i++) { var newRow = createRowCopy(getOutputRowMeta().size()); var rowIndex = getInputRowMeta().size(); // FieldName newRow[rowIndex] = fields[i]; putRow(newRow); } + FieldName + FieldName + String + -1 + -1 + N + + + 416 + 64 + Y + + + + + CleanRowSet + FilterRows + + Y + + 1 + + none + + +PickFieldName + + + + N + FieldName + IS NOT NULL + + + + + + 624 + 64 + Y + + + + + ExtractCsvMetadataToCsvWorker + MetaInject + + Y + + 1 + + none + + + rep_name + + ExtractCsvMetadataToCsvWorker + + /Library/ExtractCsvMetadataToCsv + PickMetadataFields + Name + String + -1 + -1 + Type + String + -1 + -1 + Format + String + -1 + -1 + Length + String + -1 + -1 + Precision + String + -1 + -1 + Currency + String + -1 + -1 + Decimal + String + -1 + -1 + Group + String + -1 + -1 + TrimType + String + -1 + -1 + + N + + + ReadCsv + FIELD_NAME + Y + AddFieldTypeAsString + FieldName + ReadCsv + FIELD_TYPE + Y + AddFieldTypeAsString + FieldType + + + 448 + 224 + Y + + + + + KeepHeaderOnly + FilterRows + + Y + + 1 + + none + + +BuildRowForEachField + + + + N + LineNumber + = + + constantInteger1-10N#;-# + + + + 240 + 64 + Y + + + + + PickFieldName + SelectValues + + Y + + 1 + + none + + + FieldName + + -2 + -2 + N + + + 80 + 224 + Y + + + + + ReadLinesFromCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + LineNumber + N + $[00] + $[01] +
N
+ 50000 + Y + N + N + N + + + + LineText + String + + + + + -1 + -1 + none + + + + + 80 + 64 + Y + + + + + WriteCsv + TextFileOutput + + Y + + 1 + + none + + + ${OUTPUT_DELIMITER} + ${OUTPUT_ENCLOSURE} + N + N +
Y
+
N
+ DOS + None + windows-1252 + + N + + Y + + ${OUTPUT_PATH} + N + N + N + + N + N + N + N + N + N + + Y + N + N + 0 + + + + Name + String + + + + + + none + -1 + -1 + + + Type + String + + + + + + none + -1 + -1 + + + Format + String + + + + + + none + -1 + -1 + + + Length + String + + + + + + none + -1 + -1 + + + Precision + String + + + + + + none + -1 + -1 + + + Currency + String + + + + + + none + -1 + -1 + + + Decimal + String + + + + + + none + -1 + -1 + + + Group + String + + + + + + none + -1 + -1 + + + TrimType + String + + + + + + none + -1 + -1 + + + + + 608 + 224 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr new file mode 100644 index 0000000..84af128 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr @@ -0,0 +1,462 @@ + + + ExtractCsvMetadataToCsvWorker + + + + Normal + 0 + /Library/ExtractCsvMetadataToCsv + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + + + +
+ + + +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT + + +
+ + +ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD + + +
+ +ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/22 14:58:15.112 + - + 2016/10/25 23:55:26.562 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + CleanRowSetPickMetadataFieldsY + ReadCsvDetectMetadataY + DetectMetadataCleanRowSetY + + + CleanRowSet + FilterRows + + Y + + 1 + + none + + +PickMetadataFields + + + + N + Name + IS NOT NULL + + + + + + 384 + 48 + Y + + + + + DetectMetadata + UserDefinedJavaClass + + Y + + 1 + + none + + + + + + TRANSFORM_CLASS + + Processor + + + + + + + Name + + String + + -1 + + -1 + + + + Type + + String + + -1 + + -1 + + + + Format + + String + + -1 + + -1 + + + + Length + + String + + -1 + + -1 + + + + Precision + + String + + -1 + + -1 + + + + Currency + + String + + -1 + + -1 + + + + Decimal + + String + + -1 + + -1 + + + + Group + + String + + -1 + + -1 + + + + TrimType + + String + + -1 + + -1 + + + Y + + + 240 + 48 + Y + + + + + PickMetadataFields + SelectValues + + Y + + 1 + + none + + + Name + + -2 + -2 + Type + + -2 + -2 + Format + + -2 + -2 + Length + + -2 + -2 + Precision + + -2 + -2 + Currency + + -2 + -2 + Decimal + + -2 + -2 + Group + + -2 + -2 + TrimType + + -2 + -2 + N + + + 576 + 48 + Y + + + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + 80 + 48 + Y + + + + + + + + N + + diff --git a/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb b/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb new file mode 100644 index 0000000..4a80ec0 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Library_ExportCsvToTable_Connection.kdb @@ -0,0 +1,26 @@ + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + diff --git a/pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb b/pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb new file mode 100644 index 0000000..c5ab7ed --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb @@ -0,0 +1,419 @@ + + Tester + Tester + Tester + +Dependencies: +- /Library (folder) + + 0 + /Programs/Tester + - + 2016/10/25 15:13:28.377 + - + 2017/09/14 12:05:30.241 + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 144 + 96 + + + ExportCsvInDirectoryToDatabase + + JOB + rep_name + + + ExportCsvInDirectoryToDatabase + /Library/ExportCsvInDirectoryToDatabase + N + N + N + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_DIRECTORY + + ${INPUT_DIRECTORY} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + N + Y + 0 + 336 + 96 + + + Success + + SUCCESS + N + Y + 0 + 576 + 96 + + + + + START + ExportCsvInDirectoryToDatabase + 0 + 0 + Y + Y + Y + + + ExportCsvInDirectoryToDatabase + Success + 0 + 0 + Y + Y + N + + + + + diff --git a/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr new file mode 100644 index 0000000..58250f7 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr @@ -0,0 +1,628 @@ + + + kissflow_script + + + + Normal + 0 + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2017/01/17 14:19:18.671 + - + 2017/09/15 12:00:55.538 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Get Variables + Split apps string into rows + Y + + + Create command + Execute a process + Y + + + Split apps string into rows + Create command + Y + + + + Execute a process + ExecProcess + + Y + + 1 + + none + + + command + Result output + Error output + Exit value + Y + + N + + + + + + + + + + + 800 + 96 + Y + + + + Create command + Calculator + + Y + + 1 + + none + + + + space + CONSTANT + + + + None + -1 + -1 + N + + + + + + + command + ADD3 + script_path + space + app_name + String + -1 + -1 + N + + + + + + + + + + + + + + 640 + 96 + Y + + + + Split apps string into rows + SplitFieldToRows3 + + Y + + 1 + + none + + + apps_string + , + app_name + N + + Y + N + + + + + + + + + 448 + 96 + Y + + + + Get Variables + GetVariable + + Y + + 1 + + none + + + + + apps_string + ${KISSFLOW_APPS} + String + + + + + -1 + -1 + none + + + script_path + ${KISSFLOW_SCRIPT_PATH} + String + + + + + -1 + -1 + none + + + + + + + + + + + 240 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/master_job.kjb b/pentaho/kissflow/BlocPowerETL/master_job.kjb new file mode 100644 index 0000000..fce7ae3 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/master_job.kjb @@ -0,0 +1,438 @@ + + master_job + + + + 0 + / + - + 2017/09/13 16:16:41.263 + - + 2017/09/13 16:48:42.729 + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 144 + 128 + + + Kissflow API to CSV + + TRANS + rep_name + + + kissflow_script + ${Internal.Entry.Current.Directory} + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + N + Y + 0 + 288 + 128 + + + CSV to DB + + JOB + rep_name + + + Tester + ${Internal.Entry.Current.Directory}Programs${Internal.Entry.Current.Directory}Tester + N + N + N + Y + /home/conrad/Projects/Blocpower/oltp/Kissflow/logs/kettlelog + + N + N + Debug + + Y + N + N + N + N + + Y + + N + N + Y + 0 + 448 + 128 + + + Success + + SUCCESS + N + Y + 0 + 592 + 128 + + + + + START + Kissflow API to CSV + 0 + 0 + Y + Y + Y + + + Kissflow API to CSV + CSV to DB + 0 + 0 + Y + Y + N + + + CSV to DB + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/repository.log b/pentaho/kissflow/BlocPowerETL/repository.log new file mode 100644 index 0000000..6672bd1 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/repository.log @@ -0,0 +1,10 @@ +Save database : Modification by user +Save repository element : Library_ExportCsvToTable_Connection : Modification by user +Save database : Modification by user +Save repository element : Library_ExportCsvToTable_Connection : Modification by user +Save database : Modification by user +Save repository element : Library_ExportCsvToTable_Connection : Modification by user +Save database : Modification by user +Save repository element : Library_ExportCsvToTable_Connection : Modification by user +Save database : Modification by user +Save repository element : Library_ExportCsvToTable_Connection : Modification by user diff --git a/pentaho/kissflow/ddl/create_kissflow_db..sql b/pentaho/kissflow/ddl/create_kissflow_db..sql new file mode 100644 index 0000000..1cc3b90 --- /dev/null +++ b/pentaho/kissflow/ddl/create_kissflow_db..sql @@ -0,0 +1,406 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 9.5.2 +-- Dumped by pg_dump version 9.5.3 + +DROP DATABASE IF EXISTS kissflow; +create database kissflow; + +\connect kissflow; + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; + + +-- +-- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; + + +SET search_path = public, pg_catalog; + + +SET default_with_oids = false; + +-- +-- Name: cbra; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE cbra ( + "Specific Areas of Maintenance Issues - 2" character varying(65), + "Minor (Direct Install) - 1" character varying(70), + "Specific Areas of Maintenance Issues - 1" character varying(65), + "Will HPD Finance the Project?" character varying(3), + "Received Tax Exemption?" character varying(3), + "# of Vacant Units" character varying(7), + "Client want to move forward with HPD or BlocPower?" character varying(9), + "Existing Project Debt" character varying(97), + "Leasing Challenges" character varying(54), + "Update Status in SF (Site Visit Scheduled)" boolean, + "Gas Access Code" character varying(7), + "Electricity Password" character varying(7), + "Update Status in SF (Site Visit Complete)" boolean, + "Annual Oil Cost" character varying(25), + "Project Engineer Assistant" character varying(16), + "Referral/Lead Source" character varying(13), + "Tenant Issues" character varying(150), + "Annual Water Cost" character varying(30), + "Minor (Direct Install) - 6" character varying(77), + "Type of Tax Exemption" character varying(56), + "Major (Mod Rehab) - 7" boolean, + "Are they interested in TRV installation" character varying(6), + "Major (Mod Rehab) - 8" boolean, + "Contact Phone Number" character varying(57), + "Property Type" character varying(6), + "Site Visit Necessary?" character varying(3), + "General Feedback on Client" character varying(273), + "Water Bill Acct Number" bigint, + "Was there work done?" character varying(6), + "Minor (Direct Install) - 8" character varying(48), + block bigint, + bin character varying(7), + "Major (Mod Rehab) - 6" character varying(69), + "If Moving Forward with HPD, Update Status in SF (GHPP Financing" character varying(3), + "Are they interested in a lighting upgrade?" character varying(6), + "Update Status in SF (PNS Returned from City)" character varying(3), + "Building Square Footage" bigint, + notes character varying(483), + "Maintenance Issues Notes" character varying(516), + "Update Status in SF (Schedule Site Visit)" boolean, + "Was an EEDR Created?" character varying(3), + "Create Google Calendar Invite" boolean, + "Minor (Direct Install) - 10" boolean, + "Minor (Direct Install) - 5" character varying(70), + "Contact Email Address" character varying(27), + "Bank/Bank Branch in Association" character varying(100), + "Site Contact Name and Number/Email" character varying(83), + "Water Bill" character varying(6), + "Energy and Utility Notes (Finance)" character varying(111), + "Year Built" bigint, + "Energy and Utility Notes (BizDev)" character varying(427), + "Specific Areas of Maintenance Issues - 4" character varying(65), + "Specific Areas of Maintenance Issues - 3" character varying(50), + "How did they hear about Community Retrofit/GHPP?" character varying(71), + "Financial Info Notes" character varying(352), + "Major (Mod Rehab) - 1" character varying(70), + "Are they interested in a TRP installation?" character varying(6), + "# of 4 Bedroom Units" character varying(7), + "Oil Bill" character varying(3), + "Minor (Direct Install) - 3" character varying(70), + "Is PNS Complete?" character varying(3), + "Update Status in SF (Diagnostic Report Sent to Client)" boolean, + "If Moving Forward with BlocPower for SOW, Create Project in 'CB" character varying(3), + "# of Elevators" character varying(3), + bbl bigint, + lot bigint, + "Project Engineer" character varying(9), + "Electricity Username" character varying(7), + "# of 5 Bedroom Units" character varying(7), + "If so, does the Client want to meet with HPD?" character varying(3), + "Minor (Direct Install) - 7" character varying(65), + "# of Floors" bigint, + borough character varying(8), + "Have they undertaken energy audit/assessment?" character varying(6), + "Minor (Direct Install) - 4" character varying(70), + "Minor (Direct Install) Notes" character varying(498), + "Electricity Acct Number" character varying(19), + "Operating Needs Notes (BizDev)" character varying(178), + "Major (Mod Rehab) - 10" boolean, + "Community District" character varying(4), + "Major (Mod Rehab) - 4" character varying(69), + "# of Non-Residential Units" character varying(7), + "Operating Needs Notes (Engineering)" character varying(486), + "Add Project to 'Sensors' KF App if Sensors were installed" boolean, + "Date of Assistance" timestamp without time zone, + "Gas Acct Number" character varying(11), + "Minor (Direct Install) - 2" character varying(88), + "Government Program Name" character varying(63), + "Update Status in SF (PNS Sent to City)" character varying(3), + "Gas Username" character varying(12), + "Major (Mod Rehab) Notes" character varying(474), + "Received Government Program?" character varying(6), + "Minor (Direct Install) - 9" boolean, + "Mortgage Statement Attached?" character varying(36), + "# of 0 Bedroom Units" character varying(7), + bizdev character varying(8), + "Contact Name" character varying(30), + "Operating Needs Notes (Finance)" character varying(446), + "Number of Sensors Installed" character varying(3), + "Annual Gas Cost" character varying(59), + "Site Visit Date" timestamp without time zone, + "Name of Legal Owner" character varying(76), + "# of Residential Units" bigint, + "Specific Areas of Maintenance Issues - 5" character varying(50), + "Owner Info Notes" character varying(455), + "Energy and Utility Notes (Engineering)" character varying(374), + "Annual Electricity Cost" character varying(33), + "DEP Water Rate" character varying(9), + "Heating Fuel" character varying(12), + "Type of Assistance" character varying(48), + "# of 1 Bedroom Units" character varying(7), + "Date PNS Submitted to the City" timestamp without time zone, + "Annual Taxes Paid" character varying(80), + "Have they used utility or other incentives?" character varying(6), + "Annual Heating Cost" character varying(48), + "Property Details Notes" character varying(391), + "Major (Mod Rehab) - 2" character varying(69), + "Gas Password" character varying(11), + finance character varying(10), + "Project ID" character varying(43), + "Major (Mod Rehab) - 9" boolean, + "# of 3 Bedroom Units" character varying(7), + "Annual Owner Payment" character varying(84), + "Major (Mod Rehab) - 3" character varying(69), + "Meeting w/ HPD & Client" timestamp without time zone, + "Major (Mod Rehab) - 5" character varying(44), + "# of 2 Bedroom Units" character varying(7), + "Site Address" character varying(47), + "Date Report Submitted" timestamp without time zone, + "# of Buildings" bigint, + violations character varying(350), + createdby character varying(21), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(22), + status character varying(11), + subject character varying(43), + business_step character varying(64), + business_step_1 character varying(64), + "Attachment URL" character varying(1384) +); + + +-- +-- Name: cool_roof; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE cool_roof ( + "Create google calendar invite for install" boolean, + "Update Status in SF (Schedule Inspection)" boolean, + "Cool Roof Notes" boolean, + "Update Status in SF (Complete)" boolean, + "Type of Project" character varying(4), + "Upload signed quote to Dashboard" boolean, + bbl bigint, + "Project Id" character varying(28), + "Update Status in SF (Schedule Installation)" boolean, + "Upload quotes to Dashboard" boolean, + "Update Status in SF (In Construction)" boolean, + "Update Status in SF (Waiting on Client Response)" boolean, + "Update Status in SF (Send Client Quote)" boolean, + "Update Status in SF (Bids Received/RFP Closed)" boolean, + "Update Status in SF (Out to Bid/RFP Open)" boolean, + bizdev character varying(6), + "Create Google Calendar Invite for Contractor Days" boolean, + createdby character varying(19), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(19), + status character varying(11), + subject character varying(28), + business_step character varying(20), + business_step_1 character varying(20), + "Attachment URL" boolean +); + + +-- +-- Name: lighting; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE lighting ( + "Lighting Notes" character varying(201), + "Quote Amount Post Incentives" character varying(61), + "Update Status in SF (Complete)" boolean, + "Create google calendar invite for install" character varying(2), + "Con Ed Acct Number" boolean, + "Update Status in SF (Send Client Quote)" character varying(2), + bbl bigint, + "Upload quotes to Dashboard" boolean, + "Update Status in SF (Out to Bid/RFP Open)" character varying(3), + "Upload signed quote to Dashboard" boolean, + "Update Status in SF (Waiting on Client Response)" boolean, + "Update Status in SF (Schedule Inspection)" boolean, + "Create Google Calendar Invite for Contractor Days" character varying(2), + "Quote Amount Pre Incentive" character varying(19), + "Update Status in SF (Schedule Installation)" character varying(3), + "Update Status in SF (Bids Received/RFP Closed)" boolean, + "kWh Savings on Quote" character varying(19), + "Project Id" character varying(57), + "Update Status in SF (In Construction)" character varying(3), + "Type of Project" character varying(16), + bizdev character varying(14), + createdby character varying(21), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(21), + status character varying(11), + subject character varying(57), + business_step character varying(25), + business_step_1 character varying(25), + "Attachment URL" character varying(134) +); + + +-- +-- Name: trp; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE trp ( + bbl bigint, + "Update Status in SF (Out to Bid/RFP Open)" boolean, + "TRP Notes" boolean, + "Update Status in SF (In Construction)" boolean, + "Update Status in SF (Waiting on Client Response)" boolean, + "Update Status in SF (Complete)" boolean, + "Project Id" character varying(32), + "Create Google Calendar Invite" boolean, + "Upload signed quote to Dashboard" boolean, + createdby character varying(20), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(20), + status character varying(11), + subject character varying(32), + business_step character varying(17), + business_step_1 character varying(17), + "Attachment URL" boolean +); + + +-- +-- Name: water_fixtures; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE water_fixtures ( + "Upload signed quote to Dashboard" boolean, + "Update Status in SF (In Construction)" boolean, + "Update Status in SF (Complete)" boolean, + "Project Id" character varying(40), + "Update Status in SF (Out to Bid/RFP Open)" boolean, + bizdev character varying(6), + bbl bigint, + "Create Google Calendar Invite" boolean, + "Water Fixtures Notes" character varying(169), + "Create Google Calendar Invite for Install" boolean, + "Update Status in SF (Waiting on Client Response)" boolean, + "Update Status in SF (Schedule Installation)" boolean, + createdby character varying(19), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(19), + status character varying(11), + subject character varying(40), + business_step character varying(20), + business_step_1 character varying(20), + "Attachment URL" boolean +); + + +-- +-- Name: windows; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE windows ( + "Type of Project" character varying(11), + "Update Status in SF (Send Client Quote)" boolean, + "Upload signed quote to Dashboard" boolean, + "Update Status in SF (Bids Received/RFP Closed)" boolean, + "Update Status in SF (Out to Bid/RFP Open)" boolean, + bbl character varying(3), + "Update Status in SF (Schedule Inspection)" boolean, + "Update Status in SF (Schedule Installation)" boolean, + "Update Status in SF (In Construction)" boolean, + "Windows Notes" character varying(21), + "Project Id" character varying(37), + "Update Status in SF (Complete)" boolean, + "Create Google Calendar Invite for Contractor Days" boolean, + "Update Status in SF (Waiting on Client Response)" boolean, + "Upload quotes to Dashboard" boolean, + "Create google calendar invite for install" boolean, + bizdev character varying(6), + createdby character varying(20), + createdat timestamp without time zone, + modifiedat timestamp without time zone, + modifiedby character varying(20), + status character varying(11), + subject character varying(37), + business_step character varying(20), + business_step_1 character varying(20), + "Attachment URL" boolean +); + + +-- +-- Name: all_kissflow_projects; Type: VIEW; Schema: public; Owner: - +-- + +CREATE VIEW all_kissflow_projects AS + SELECT cbra."Project ID" AS kf_project_id, + (cbra.bbl)::text AS bbl, + 'CBRA'::text AS kissflow_form_type, + cbra.status, + cbra.business_step, + cbra.business_step_1 + FROM cbra +UNION + SELECT cool_roof."Project Id" AS kf_project_id, + (cool_roof.bbl)::text AS bbl, + 'COOL ROOF'::text AS kissflow_form_type, + cool_roof.status, + cool_roof.business_step, + cool_roof.business_step_1 + FROM cool_roof +UNION + SELECT lighting."Project Id" AS kf_project_id, + (lighting.bbl)::text AS bbl, + 'LIGHTING'::text AS kissflow_form_type, + lighting.status, + lighting.business_step, + lighting.business_step_1 + FROM lighting +UNION + SELECT trp."Project Id" AS kf_project_id, + (trp.bbl)::text AS bbl, + 'TRP'::text AS kissflow_form_type, + trp.status, + trp.business_step, + trp.business_step_1 + FROM trp +UNION + SELECT water_fixtures."Project Id" AS kf_project_id, + (water_fixtures.bbl)::text AS bbl, + 'WATER FIXTURES'::text AS kissflow_form_type, + water_fixtures.status, + water_fixtures.business_step, + water_fixtures.business_step_1 + FROM water_fixtures +UNION + SELECT windows."Project Id" AS kf_project_id, + (windows.bbl)::text AS bbl, + 'WINDOWS'::text AS kissflow_form_type, + windows.status, + windows.business_step, + windows.business_step_1 + FROM windows; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py new file mode 100644 index 0000000..2d8f54e --- /dev/null +++ b/pentaho/kissflow/kissflow.py @@ -0,0 +1,84 @@ +from datetime import datetime +import csv +import json +import requests +import sys +import time + +LOGGING_PATH = '/home/conrad/Projects/Blocpower/oltp/Kissflow/logs/pylog-{}' +CSV_PATH = '/home/conrad/Projects/Blocpower/oltp/Kissflow/kissflow_apps/app_{}.csv' +KISSFLOW_API_KEY = '9b7b95b8-d749-11e6-8eba-2d38b0e26d4e' +KISSFLOW_API_URL = 'https://kf-0003173.appspot.com/api/1/{}/list/p{}/{}' +APP_NAME = sys.argv[1] +# If testing is true, only grab first 50 entries +TESTING = True +PAGE_SIZE = 50 +start = time.time() + +log_file = open(LOGGING_PATH.format(APP_NAME), 'w') +def log(s): + log_file.write('{}: {}\n'.format( + str(datetime.now()), + str(s), + )) + log_file.flush() + +log('Starting the script for app {}'.format(APP_NAME)) + +page_num = 1 +total_results = [] +while True: + response = requests.get(KISSFLOW_API_URL.format( + APP_NAME.replace('_', ' '), + page_num, + PAGE_SIZE, + ), headers={'api_key': KISSFLOW_API_KEY}) + if response.status_code != 200: + log('Error while making a request to {}'.format( + KISSFLOW_API_URL.format(APP_NAME, page_num, PAGE_SIZE) + )) + log('Status code was {}'.format(response.status_code)) + sys.exit(0) + json_result = response.json() + total_results.extend(json_result) + if len(json_result) < PAGE_SIZE: + break + + page_num += 1 + log('{} results so far...'.format(len(total_results))) + if TESTING: + break + +log('Get request succeeded with {} total requests retrieved'.format(len(total_results))) + +column_name_list = set() +for result in total_results: + for key in result: + column_name_list.add(key.encode('ascii', 'ignore')) +column_name_list = list(column_name_list) + +log('{} total unique column names'.format(len(column_name_list))) + +rowCounter = 1 +with open(CSV_PATH.format(APP_NAME), 'wb') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(column_name_list) + for result in total_results: + if rowCounter % 100 == 0: + log('Created {} rows so far..'.format(rowCounter)) + row = [] + for column in column_name_list: + if column in result: + field = result[column] + if type(field) is list: + field = json.dumps(field) + elif type(field) is not str: + field = str(field) + + row.append(field.encode('ascii', 'ignore')) + else: + row.append(None) + csvwriter.writerow(row) + rowCounter += 1 +end = time.time() +log('Completed script in {} seconds'.format(end - start)) diff --git a/pentaho/kissflow/kissflow_transformation.ktr b/pentaho/kissflow/kissflow_transformation.ktr new file mode 100644 index 0000000..a2afe44 --- /dev/null +++ b/pentaho/kissflow/kissflow_transformation.ktr @@ -0,0 +1,526 @@ + + + + Kissflow_DI_GET + + + + Normal + 0 + / + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2017/01/17 14:19:18.671 + - + 2017/01/17 14:19:18.671 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + Data Grid + Execute a process + Y + + + + Data Grid + DataGrid + + Y + + 1 + + none + + + + + command + String + + + + + -1 + -1 + N + + + + + /home/conrad/Projects/Blocpower/oltp/Kissflow/run_kissflow.sh + + + + + + + + + + + 96 + 96 + Y + + + + Execute a process + ExecProcess + + Y + + 1 + + none + + + command + Result output + Error output + Exit value + Y + + N + + + + + + + + + + + 256 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/run_kissflow.sh b/pentaho/kissflow/run_kissflow.sh new file mode 100755 index 0000000..77df118 --- /dev/null +++ b/pentaho/kissflow/run_kissflow.sh @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/python /home/conrad/Projects/Blocpower/oltp/Kissflow/kissflow.py $1 diff --git a/pentaho/kissflow/scripts/RestAPI data Retrieve Script b/pentaho/kissflow/scripts/RestAPI data Retrieve Script new file mode 100644 index 0000000..72a0004 --- /dev/null +++ b/pentaho/kissflow/scripts/RestAPI data Retrieve Script @@ -0,0 +1,32 @@ +#!/bin/sh + +# RestAPI update Script.sh +# +# +# Created by Yusuf Liu on 1/10/17. +# +# purpose of this script is to update the records in kissflow + +record="999_test_999" +APIKEY="9b7b95b8-d749-11e6-8eba-2d38b0e26d4e" +AccountID="kf-0003173" +project="CBRA" + +# connect to the kissflow api +APIanswer=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/verify ) +checkmsg='{"success": "All O.K."}' + +# error handling +if [ "$APIanswer" = "$checkmsg" ] +then + echo "$APIanswer" +else + echo "connection failed" +fi + + +response=$(curl -s -H "api_key:$APIKEY" -X POST --data-urlencode "address=$record" https://kf-0003173.appspot.com/api/1/di_dev/submit) +response=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/CBRA/list/p1/2 ) +response2=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/CBRA/list/Initiated_On/p2/1 ) +echo "$response" +echo "$response2" diff --git a/pentaho/kissflow/scripts/RestAPI update Script.sh b/pentaho/kissflow/scripts/RestAPI update Script.sh new file mode 100644 index 0000000..533b0a3 --- /dev/null +++ b/pentaho/kissflow/scripts/RestAPI update Script.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# RestAPI update Script.sh +# +# +# Created by Yusuf Liu on 1/10/17. +# +# purpose of this script is to update the records in kissflow + +record="999_test_999" +APIKEY="9b7b95b8-d749-11e6-8eba-2d38b0e26d4e" +AccountID="kf-0003173" + +# connect to the kissflow api +APIanswer=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/verify ) +checkmsg='{"success": "All O.K."}' + +# error handling +if [ "$APIanswer" = "$checkmsg" ] +then + echo "$APIanswer" +else + echo "connection failed" +fi + + +response=$(curl -s -H "api_key:$APIKEY" -X POST --data-urlencode "address=$record" https://kf-0003173.appspot.com/api/1/di_dev/submit) + +echo "$response" -- GitLab From f50fa21ca0896e4ee1c487f15164e139594cd304 Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 Sep 2017 18:07:28 -0400 Subject: [PATCH 02/11] Add repository.xml --- Dockerfile | 9 +- README.md | 5 + config/carte_master_config.xml | 3 + config/repositories.xml | 12 + pentaho/kissflow/ddl/create_kissflow_db..sql | 406 -------------- pentaho/kissflow/kissflow.py | 11 +- pentaho/kissflow/kissflow_transformation.ktr | 526 ------------------ pentaho/kissflow/run_kissflow.sh | 2 +- .../scripts/RestAPI data Retrieve Script | 32 -- .../kissflow/scripts/RestAPI update Script.sh | 29 - 10 files changed, 35 insertions(+), 1000 deletions(-) create mode 100644 config/repositories.xml delete mode 100644 pentaho/kissflow/ddl/create_kissflow_db..sql delete mode 100644 pentaho/kissflow/kissflow_transformation.ktr delete mode 100644 pentaho/kissflow/scripts/RestAPI data Retrieve Script delete mode 100644 pentaho/kissflow/scripts/RestAPI update Script.sh diff --git a/Dockerfile b/Dockerfile index 625d325..e5f2fcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,11 @@ ENV PDI_RELEASE=7.1 \ KETTLE_HOME=/pentaho-di RUN apt-get update && apt-get install -y \ - libwebkitgtk-1.0.0 + libwebkitgtk-1.0.0 \ + python \ + python-pip + +RUN pip install requests RUN curl -L -o /tmp/pdi-ce-${PDI_VERSION}.zip \ https://superb-dca2.dl.sourceforge.net/project/pentaho/Data%20Integration/${PDI_RELEASE}/pdi-ce-${PDI_VERSION}.zip && \ @@ -16,8 +20,11 @@ RUN curl -L -o /tmp/pdi-ce-${PDI_VERSION}.zip \ ENV PATH=$PDI_HOME/data-integration:$PATH RUN mkdir -p $KETTLE_HOME/.kettle +RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ +COPY config/repositories.xml $KETTLE_HOME/.kettle/repositories.xml +COPY pentaho/ $KETTLE_HOME/jobs/ COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/README.md b/README.md index 28286a9..e01e50e 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,8 @@ CARTE_MASTER_PASSWORD ``` Variables for `kettle.properties` must be prefixed with `KP_`. + + +curl -X POST 'localhost:9081/kettle/executeJob/?job=/pentaho-di/jobs/kissflow/BlocPowerETL/master_job.kjb&level=Debug' --user conrad:password +docker run -p 9081:9081 --env-file .env pentaho +docker build -t pentaho . diff --git a/config/carte_master_config.xml b/config/carte_master_config.xml index 97798e2..52545ca 100644 --- a/config/carte_master_config.xml +++ b/config/carte_master_config.xml @@ -7,6 +7,9 @@ CARTE_MASTER_USER CARTE_MASTER_PASSWORD + + kissflow + 10000 1440 diff --git a/config/repositories.xml b/config/repositories.xml new file mode 100644 index 0000000..6869b79 --- /dev/null +++ b/config/repositories.xml @@ -0,0 +1,12 @@ + + + KettleFileRepository + kissflow + File repository + true + /pentaho-di/jobs/kissflow/BlocPowerETL/ + N + N + + + diff --git a/pentaho/kissflow/ddl/create_kissflow_db..sql b/pentaho/kissflow/ddl/create_kissflow_db..sql deleted file mode 100644 index 1cc3b90..0000000 --- a/pentaho/kissflow/ddl/create_kissflow_db..sql +++ /dev/null @@ -1,406 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 9.5.2 --- Dumped by pg_dump version 9.5.3 - -DROP DATABASE IF EXISTS kissflow; -create database kissflow; - -\connect kissflow; - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SET check_function_bodies = false; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - - --- --- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; - - -SET search_path = public, pg_catalog; - - -SET default_with_oids = false; - --- --- Name: cbra; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE cbra ( - "Specific Areas of Maintenance Issues - 2" character varying(65), - "Minor (Direct Install) - 1" character varying(70), - "Specific Areas of Maintenance Issues - 1" character varying(65), - "Will HPD Finance the Project?" character varying(3), - "Received Tax Exemption?" character varying(3), - "# of Vacant Units" character varying(7), - "Client want to move forward with HPD or BlocPower?" character varying(9), - "Existing Project Debt" character varying(97), - "Leasing Challenges" character varying(54), - "Update Status in SF (Site Visit Scheduled)" boolean, - "Gas Access Code" character varying(7), - "Electricity Password" character varying(7), - "Update Status in SF (Site Visit Complete)" boolean, - "Annual Oil Cost" character varying(25), - "Project Engineer Assistant" character varying(16), - "Referral/Lead Source" character varying(13), - "Tenant Issues" character varying(150), - "Annual Water Cost" character varying(30), - "Minor (Direct Install) - 6" character varying(77), - "Type of Tax Exemption" character varying(56), - "Major (Mod Rehab) - 7" boolean, - "Are they interested in TRV installation" character varying(6), - "Major (Mod Rehab) - 8" boolean, - "Contact Phone Number" character varying(57), - "Property Type" character varying(6), - "Site Visit Necessary?" character varying(3), - "General Feedback on Client" character varying(273), - "Water Bill Acct Number" bigint, - "Was there work done?" character varying(6), - "Minor (Direct Install) - 8" character varying(48), - block bigint, - bin character varying(7), - "Major (Mod Rehab) - 6" character varying(69), - "If Moving Forward with HPD, Update Status in SF (GHPP Financing" character varying(3), - "Are they interested in a lighting upgrade?" character varying(6), - "Update Status in SF (PNS Returned from City)" character varying(3), - "Building Square Footage" bigint, - notes character varying(483), - "Maintenance Issues Notes" character varying(516), - "Update Status in SF (Schedule Site Visit)" boolean, - "Was an EEDR Created?" character varying(3), - "Create Google Calendar Invite" boolean, - "Minor (Direct Install) - 10" boolean, - "Minor (Direct Install) - 5" character varying(70), - "Contact Email Address" character varying(27), - "Bank/Bank Branch in Association" character varying(100), - "Site Contact Name and Number/Email" character varying(83), - "Water Bill" character varying(6), - "Energy and Utility Notes (Finance)" character varying(111), - "Year Built" bigint, - "Energy and Utility Notes (BizDev)" character varying(427), - "Specific Areas of Maintenance Issues - 4" character varying(65), - "Specific Areas of Maintenance Issues - 3" character varying(50), - "How did they hear about Community Retrofit/GHPP?" character varying(71), - "Financial Info Notes" character varying(352), - "Major (Mod Rehab) - 1" character varying(70), - "Are they interested in a TRP installation?" character varying(6), - "# of 4 Bedroom Units" character varying(7), - "Oil Bill" character varying(3), - "Minor (Direct Install) - 3" character varying(70), - "Is PNS Complete?" character varying(3), - "Update Status in SF (Diagnostic Report Sent to Client)" boolean, - "If Moving Forward with BlocPower for SOW, Create Project in 'CB" character varying(3), - "# of Elevators" character varying(3), - bbl bigint, - lot bigint, - "Project Engineer" character varying(9), - "Electricity Username" character varying(7), - "# of 5 Bedroom Units" character varying(7), - "If so, does the Client want to meet with HPD?" character varying(3), - "Minor (Direct Install) - 7" character varying(65), - "# of Floors" bigint, - borough character varying(8), - "Have they undertaken energy audit/assessment?" character varying(6), - "Minor (Direct Install) - 4" character varying(70), - "Minor (Direct Install) Notes" character varying(498), - "Electricity Acct Number" character varying(19), - "Operating Needs Notes (BizDev)" character varying(178), - "Major (Mod Rehab) - 10" boolean, - "Community District" character varying(4), - "Major (Mod Rehab) - 4" character varying(69), - "# of Non-Residential Units" character varying(7), - "Operating Needs Notes (Engineering)" character varying(486), - "Add Project to 'Sensors' KF App if Sensors were installed" boolean, - "Date of Assistance" timestamp without time zone, - "Gas Acct Number" character varying(11), - "Minor (Direct Install) - 2" character varying(88), - "Government Program Name" character varying(63), - "Update Status in SF (PNS Sent to City)" character varying(3), - "Gas Username" character varying(12), - "Major (Mod Rehab) Notes" character varying(474), - "Received Government Program?" character varying(6), - "Minor (Direct Install) - 9" boolean, - "Mortgage Statement Attached?" character varying(36), - "# of 0 Bedroom Units" character varying(7), - bizdev character varying(8), - "Contact Name" character varying(30), - "Operating Needs Notes (Finance)" character varying(446), - "Number of Sensors Installed" character varying(3), - "Annual Gas Cost" character varying(59), - "Site Visit Date" timestamp without time zone, - "Name of Legal Owner" character varying(76), - "# of Residential Units" bigint, - "Specific Areas of Maintenance Issues - 5" character varying(50), - "Owner Info Notes" character varying(455), - "Energy and Utility Notes (Engineering)" character varying(374), - "Annual Electricity Cost" character varying(33), - "DEP Water Rate" character varying(9), - "Heating Fuel" character varying(12), - "Type of Assistance" character varying(48), - "# of 1 Bedroom Units" character varying(7), - "Date PNS Submitted to the City" timestamp without time zone, - "Annual Taxes Paid" character varying(80), - "Have they used utility or other incentives?" character varying(6), - "Annual Heating Cost" character varying(48), - "Property Details Notes" character varying(391), - "Major (Mod Rehab) - 2" character varying(69), - "Gas Password" character varying(11), - finance character varying(10), - "Project ID" character varying(43), - "Major (Mod Rehab) - 9" boolean, - "# of 3 Bedroom Units" character varying(7), - "Annual Owner Payment" character varying(84), - "Major (Mod Rehab) - 3" character varying(69), - "Meeting w/ HPD & Client" timestamp without time zone, - "Major (Mod Rehab) - 5" character varying(44), - "# of 2 Bedroom Units" character varying(7), - "Site Address" character varying(47), - "Date Report Submitted" timestamp without time zone, - "# of Buildings" bigint, - violations character varying(350), - createdby character varying(21), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(22), - status character varying(11), - subject character varying(43), - business_step character varying(64), - business_step_1 character varying(64), - "Attachment URL" character varying(1384) -); - - --- --- Name: cool_roof; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE cool_roof ( - "Create google calendar invite for install" boolean, - "Update Status in SF (Schedule Inspection)" boolean, - "Cool Roof Notes" boolean, - "Update Status in SF (Complete)" boolean, - "Type of Project" character varying(4), - "Upload signed quote to Dashboard" boolean, - bbl bigint, - "Project Id" character varying(28), - "Update Status in SF (Schedule Installation)" boolean, - "Upload quotes to Dashboard" boolean, - "Update Status in SF (In Construction)" boolean, - "Update Status in SF (Waiting on Client Response)" boolean, - "Update Status in SF (Send Client Quote)" boolean, - "Update Status in SF (Bids Received/RFP Closed)" boolean, - "Update Status in SF (Out to Bid/RFP Open)" boolean, - bizdev character varying(6), - "Create Google Calendar Invite for Contractor Days" boolean, - createdby character varying(19), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(19), - status character varying(11), - subject character varying(28), - business_step character varying(20), - business_step_1 character varying(20), - "Attachment URL" boolean -); - - --- --- Name: lighting; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE lighting ( - "Lighting Notes" character varying(201), - "Quote Amount Post Incentives" character varying(61), - "Update Status in SF (Complete)" boolean, - "Create google calendar invite for install" character varying(2), - "Con Ed Acct Number" boolean, - "Update Status in SF (Send Client Quote)" character varying(2), - bbl bigint, - "Upload quotes to Dashboard" boolean, - "Update Status in SF (Out to Bid/RFP Open)" character varying(3), - "Upload signed quote to Dashboard" boolean, - "Update Status in SF (Waiting on Client Response)" boolean, - "Update Status in SF (Schedule Inspection)" boolean, - "Create Google Calendar Invite for Contractor Days" character varying(2), - "Quote Amount Pre Incentive" character varying(19), - "Update Status in SF (Schedule Installation)" character varying(3), - "Update Status in SF (Bids Received/RFP Closed)" boolean, - "kWh Savings on Quote" character varying(19), - "Project Id" character varying(57), - "Update Status in SF (In Construction)" character varying(3), - "Type of Project" character varying(16), - bizdev character varying(14), - createdby character varying(21), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(21), - status character varying(11), - subject character varying(57), - business_step character varying(25), - business_step_1 character varying(25), - "Attachment URL" character varying(134) -); - - --- --- Name: trp; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE trp ( - bbl bigint, - "Update Status in SF (Out to Bid/RFP Open)" boolean, - "TRP Notes" boolean, - "Update Status in SF (In Construction)" boolean, - "Update Status in SF (Waiting on Client Response)" boolean, - "Update Status in SF (Complete)" boolean, - "Project Id" character varying(32), - "Create Google Calendar Invite" boolean, - "Upload signed quote to Dashboard" boolean, - createdby character varying(20), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(20), - status character varying(11), - subject character varying(32), - business_step character varying(17), - business_step_1 character varying(17), - "Attachment URL" boolean -); - - --- --- Name: water_fixtures; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE water_fixtures ( - "Upload signed quote to Dashboard" boolean, - "Update Status in SF (In Construction)" boolean, - "Update Status in SF (Complete)" boolean, - "Project Id" character varying(40), - "Update Status in SF (Out to Bid/RFP Open)" boolean, - bizdev character varying(6), - bbl bigint, - "Create Google Calendar Invite" boolean, - "Water Fixtures Notes" character varying(169), - "Create Google Calendar Invite for Install" boolean, - "Update Status in SF (Waiting on Client Response)" boolean, - "Update Status in SF (Schedule Installation)" boolean, - createdby character varying(19), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(19), - status character varying(11), - subject character varying(40), - business_step character varying(20), - business_step_1 character varying(20), - "Attachment URL" boolean -); - - --- --- Name: windows; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE windows ( - "Type of Project" character varying(11), - "Update Status in SF (Send Client Quote)" boolean, - "Upload signed quote to Dashboard" boolean, - "Update Status in SF (Bids Received/RFP Closed)" boolean, - "Update Status in SF (Out to Bid/RFP Open)" boolean, - bbl character varying(3), - "Update Status in SF (Schedule Inspection)" boolean, - "Update Status in SF (Schedule Installation)" boolean, - "Update Status in SF (In Construction)" boolean, - "Windows Notes" character varying(21), - "Project Id" character varying(37), - "Update Status in SF (Complete)" boolean, - "Create Google Calendar Invite for Contractor Days" boolean, - "Update Status in SF (Waiting on Client Response)" boolean, - "Upload quotes to Dashboard" boolean, - "Create google calendar invite for install" boolean, - bizdev character varying(6), - createdby character varying(20), - createdat timestamp without time zone, - modifiedat timestamp without time zone, - modifiedby character varying(20), - status character varying(11), - subject character varying(37), - business_step character varying(20), - business_step_1 character varying(20), - "Attachment URL" boolean -); - - --- --- Name: all_kissflow_projects; Type: VIEW; Schema: public; Owner: - --- - -CREATE VIEW all_kissflow_projects AS - SELECT cbra."Project ID" AS kf_project_id, - (cbra.bbl)::text AS bbl, - 'CBRA'::text AS kissflow_form_type, - cbra.status, - cbra.business_step, - cbra.business_step_1 - FROM cbra -UNION - SELECT cool_roof."Project Id" AS kf_project_id, - (cool_roof.bbl)::text AS bbl, - 'COOL ROOF'::text AS kissflow_form_type, - cool_roof.status, - cool_roof.business_step, - cool_roof.business_step_1 - FROM cool_roof -UNION - SELECT lighting."Project Id" AS kf_project_id, - (lighting.bbl)::text AS bbl, - 'LIGHTING'::text AS kissflow_form_type, - lighting.status, - lighting.business_step, - lighting.business_step_1 - FROM lighting -UNION - SELECT trp."Project Id" AS kf_project_id, - (trp.bbl)::text AS bbl, - 'TRP'::text AS kissflow_form_type, - trp.status, - trp.business_step, - trp.business_step_1 - FROM trp -UNION - SELECT water_fixtures."Project Id" AS kf_project_id, - (water_fixtures.bbl)::text AS bbl, - 'WATER FIXTURES'::text AS kissflow_form_type, - water_fixtures.status, - water_fixtures.business_step, - water_fixtures.business_step_1 - FROM water_fixtures -UNION - SELECT windows."Project Id" AS kf_project_id, - (windows.bbl)::text AS bbl, - 'WINDOWS'::text AS kissflow_form_type, - windows.status, - windows.business_step, - windows.business_step_1 - FROM windows; - - --- --- PostgreSQL database dump complete --- - diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py index 2d8f54e..b596d47 100644 --- a/pentaho/kissflow/kissflow.py +++ b/pentaho/kissflow/kissflow.py @@ -4,18 +4,19 @@ import json import requests import sys import time +import os -LOGGING_PATH = '/home/conrad/Projects/Blocpower/oltp/Kissflow/logs/pylog-{}' -CSV_PATH = '/home/conrad/Projects/Blocpower/oltp/Kissflow/kissflow_apps/app_{}.csv' -KISSFLOW_API_KEY = '9b7b95b8-d749-11e6-8eba-2d38b0e26d4e' -KISSFLOW_API_URL = 'https://kf-0003173.appspot.com/api/1/{}/list/p{}/{}' +LOGGING_PATH = os.environ['KISSFLOW_LOGGING_PATH'] +CSV_PATH = os.environ['KISSFLOW_CSV_PATH'] +KISSFLOW_API_KEY = os.environ['KISSFLOW_API_KEY'] +KISSFLOW_API_URL = os.environ['KISSFLOW_API_URL'] APP_NAME = sys.argv[1] # If testing is true, only grab first 50 entries TESTING = True PAGE_SIZE = 50 start = time.time() -log_file = open(LOGGING_PATH.format(APP_NAME), 'w') +log_file = open(LOGGING_PATH.format(APP_NAME), 'a') def log(s): log_file.write('{}: {}\n'.format( str(datetime.now()), diff --git a/pentaho/kissflow/kissflow_transformation.ktr b/pentaho/kissflow/kissflow_transformation.ktr deleted file mode 100644 index a2afe44..0000000 --- a/pentaho/kissflow/kissflow_transformation.ktr +++ /dev/null @@ -1,526 +0,0 @@ - - - - Kissflow_DI_GET - - - - Normal - 0 - / - - - - - - -
- - - - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - TRANSNAME - Y - TRANSNAME - - - STATUS - Y - STATUS - - - LINES_READ - Y - LINES_READ - - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - - LINES_UPDATED - Y - LINES_UPDATED - - - - LINES_INPUT - Y - LINES_INPUT - - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - - LINES_REJECTED - Y - LINES_REJECTED - - - - ERRORS - Y - ERRORS - - - STARTDATE - Y - STARTDATE - - - ENDDATE - Y - ENDDATE - - - LOGDATE - Y - LOGDATE - - - DEPDATE - Y - DEPDATE - - - REPLAYDATE - Y - REPLAYDATE - - - LOG_FIELD - Y - LOG_FIELD - - - EXECUTING_SERVER - N - EXECUTING_SERVER - - - EXECUTING_USER - N - EXECUTING_USER - - - CLIENT - N - CLIENT - - - - - -
- - - - ID_BATCH - Y - ID_BATCH - - - SEQ_NR - Y - SEQ_NR - - - LOGDATE - Y - LOGDATE - - - TRANSNAME - Y - TRANSNAME - - - STEPNAME - Y - STEPNAME - - - STEP_COPY - Y - STEP_COPY - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - INPUT_BUFFER_ROWS - Y - INPUT_BUFFER_ROWS - - - OUTPUT_BUFFER_ROWS - Y - OUTPUT_BUFFER_ROWS - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - LOGGING_OBJECT_TYPE - Y - LOGGING_OBJECT_TYPE - - - OBJECT_NAME - Y - OBJECT_NAME - - - OBJECT_COPY - Y - OBJECT_COPY - - - REPOSITORY_DIRECTORY - Y - REPOSITORY_DIRECTORY - - - FILENAME - Y - FILENAME - - - OBJECT_ID - Y - OBJECT_ID - - - OBJECT_REVISION - Y - OBJECT_REVISION - - - PARENT_CHANNEL_ID - Y - PARENT_CHANNEL_ID - - - ROOT_CHANNEL_ID - Y - ROOT_CHANNEL_ID - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - TRANSNAME - Y - TRANSNAME - - - STEPNAME - Y - STEPNAME - - - STEP_COPY - Y - STEP_COPY - - - LINES_READ - Y - LINES_READ - - - LINES_WRITTEN - Y - LINES_WRITTEN - - - LINES_UPDATED - Y - LINES_UPDATED - - - LINES_INPUT - Y - LINES_INPUT - - - LINES_OUTPUT - Y - LINES_OUTPUT - - - LINES_REJECTED - Y - LINES_REJECTED - - - ERRORS - Y - ERRORS - - - LOG_FIELD - N - LOG_FIELD - - - - - -
- - - ID_BATCH - Y - ID_BATCH - - - CHANNEL_ID - Y - CHANNEL_ID - - - LOG_DATE - Y - LOG_DATE - - - METRICS_DATE - Y - METRICS_DATE - - - METRICS_CODE - Y - METRICS_CODE - - - METRICS_DESCRIPTION - Y - METRICS_DESCRIPTION - - - METRICS_SUBJECT - Y - METRICS_SUBJECT - - - METRICS_TYPE - Y - METRICS_TYPE - - - METRICS_VALUE - Y - METRICS_VALUE - - - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2017/01/17 14:19:18.671 - - - 2017/01/17 14:19:18.671 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - - Data Grid - Execute a process - Y - - - - Data Grid - DataGrid - - Y - - 1 - - none - - - - - command - String - - - - - -1 - -1 - N - - - - - /home/conrad/Projects/Blocpower/oltp/Kissflow/run_kissflow.sh - - - - - - - - - - - 96 - 96 - Y - - - - Execute a process - ExecProcess - - Y - - 1 - - none - - - command - Result output - Error output - Exit value - Y - - N - - - - - - - - - - - 256 - 96 - Y - - - - - - - N - diff --git a/pentaho/kissflow/run_kissflow.sh b/pentaho/kissflow/run_kissflow.sh index 77df118..93c6715 100755 --- a/pentaho/kissflow/run_kissflow.sh +++ b/pentaho/kissflow/run_kissflow.sh @@ -1,2 +1,2 @@ #!/bin/bash -/usr/bin/python /home/conrad/Projects/Blocpower/oltp/Kissflow/kissflow.py $1 +/usr/bin/python /pentaho-di/jobs/kissflow/kissflow.py $1 diff --git a/pentaho/kissflow/scripts/RestAPI data Retrieve Script b/pentaho/kissflow/scripts/RestAPI data Retrieve Script deleted file mode 100644 index 72a0004..0000000 --- a/pentaho/kissflow/scripts/RestAPI data Retrieve Script +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# RestAPI update Script.sh -# -# -# Created by Yusuf Liu on 1/10/17. -# -# purpose of this script is to update the records in kissflow - -record="999_test_999" -APIKEY="9b7b95b8-d749-11e6-8eba-2d38b0e26d4e" -AccountID="kf-0003173" -project="CBRA" - -# connect to the kissflow api -APIanswer=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/verify ) -checkmsg='{"success": "All O.K."}' - -# error handling -if [ "$APIanswer" = "$checkmsg" ] -then - echo "$APIanswer" -else - echo "connection failed" -fi - - -response=$(curl -s -H "api_key:$APIKEY" -X POST --data-urlencode "address=$record" https://kf-0003173.appspot.com/api/1/di_dev/submit) -response=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/CBRA/list/p1/2 ) -response2=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/CBRA/list/Initiated_On/p2/1 ) -echo "$response" -echo "$response2" diff --git a/pentaho/kissflow/scripts/RestAPI update Script.sh b/pentaho/kissflow/scripts/RestAPI update Script.sh deleted file mode 100644 index 533b0a3..0000000 --- a/pentaho/kissflow/scripts/RestAPI update Script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# RestAPI update Script.sh -# -# -# Created by Yusuf Liu on 1/10/17. -# -# purpose of this script is to update the records in kissflow - -record="999_test_999" -APIKEY="9b7b95b8-d749-11e6-8eba-2d38b0e26d4e" -AccountID="kf-0003173" - -# connect to the kissflow api -APIanswer=$(curl -s -H "api_key:$APIKEY" -X GET https://kf-0003173.appspot.com/api/1/verify ) -checkmsg='{"success": "All O.K."}' - -# error handling -if [ "$APIanswer" = "$checkmsg" ] -then - echo "$APIanswer" -else - echo "connection failed" -fi - - -response=$(curl -s -H "api_key:$APIKEY" -X POST --data-urlencode "address=$record" https://kf-0003173.appspot.com/api/1/di_dev/submit) - -echo "$response" -- GitLab From 481923b76387070e4ef1d3064df8a9f10c45edd7 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 Sep 2017 15:47:42 -0400 Subject: [PATCH 03/11] Fix errors with being a repository --- Dockerfile | 4 +- config/repositories.xml | 12 - config/shared.xml | 30 + .../BlocPowerETL/DefineSqlVariables.ktr | 749 +++++++++++ .../BlocPowerETL/DefineSqlVariablesWorker.ktr | 716 +++++++++++ .../ExportCsvInDirectoryToDatabase.kjb | 566 +++++++++ .../BlocPowerETL/ExportCsvToTable.kjb | 887 +++++++++++++ .../BlocPowerETL/ExportCsvToTable.ktr | 749 +++++++++++ .../ExportCsvToTableWorker.ktr | 0 .../BlocPowerETL/ExtractCsvMetadataToCsv.kjb | 460 +++++++ .../BlocPowerETL/ExtractCsvMetadataToCsv.ktr | 1116 +++++++++++++++++ .../ExtractCsvMetadataToCsvWorker.ktr | 0 .../GetCsvInDirectoryInformation.ktr | 645 ++++++++++ .../ExportCsvInDirectoryToDatabase.kjb | 256 ---- .../GetCsvInDirectoryInformation.ktr | 239 ---- .../ExportCsvToTable/DefineSqlVariables.ktr | 353 ------ .../DefineSqlVariablesWorker.ktr | 323 ----- .../ExportCsvToTable/ExportCsvToTable.kjb | 491 -------- .../ExportCsvToTable/ExportCsvToTable.ktr | 353 ------ .../ExtractCsvMetadataToCsv.kjb | 172 --- .../ExtractCsvMetadataToCsv.ktr | 550 -------- .../{Programs/Tester => }/Tester.kjb | 10 +- .../kissflow/BlocPowerETL/kissflow_script.ktr | 87 +- pentaho/kissflow/BlocPowerETL/master_job.kjb | 16 +- pentaho/kissflow/kissflow.py | 1 + 25 files changed, 5979 insertions(+), 2806 deletions(-) delete mode 100644 config/repositories.xml create mode 100644 config/shared.xml create mode 100644 pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr create mode 100644 pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr rename pentaho/kissflow/BlocPowerETL/{Library/ExportCsvToTable => }/ExportCsvToTableWorker.ktr (100%) create mode 100644 pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb create mode 100644 pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr rename pentaho/kissflow/BlocPowerETL/{Library/ExtractCsvMetadataToCsv => }/ExtractCsvMetadataToCsvWorker.ktr (100%) create mode 100644 pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb delete mode 100644 pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr rename pentaho/kissflow/BlocPowerETL/{Programs/Tester => }/Tester.kjb (97%) mode change 100644 => 100755 pentaho/kissflow/kissflow.py diff --git a/Dockerfile b/Dockerfile index e5f2fcf..e9ea9f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,8 @@ ENV PDI_RELEASE=7.1 \ RUN apt-get update && apt-get install -y \ libwebkitgtk-1.0.0 \ python \ - python-pip + python-pip \ + vim RUN pip install requests @@ -24,6 +25,7 @@ RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ COPY config/repositories.xml $KETTLE_HOME/.kettle/repositories.xml +COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml COPY pentaho/ $KETTLE_HOME/jobs/ COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / diff --git a/config/repositories.xml b/config/repositories.xml deleted file mode 100644 index 6869b79..0000000 --- a/config/repositories.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - KettleFileRepository - kissflow - File repository - true - /pentaho-di/jobs/kissflow/BlocPowerETL/ - N - N - - - diff --git a/config/shared.xml b/config/shared.xml new file mode 100644 index 0000000..8439711 --- /dev/null +++ b/config/shared.xml @@ -0,0 +1,30 @@ + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} + CUSTOM_URL${OUTPUT_URL} + FORCE_IDENTIFIERS_TO_LOWERCASEN + FORCE_IDENTIFIERS_TO_UPPERCASEN + IS_CLUSTEREDN + PORT_NUMBER1521 + PRESERVE_RESERVED_WORD_CASEY + QUOTE_ALL_FIELDSN + SUPPORTS_BOOLEAN_DATA_TYPEY + SUPPORTS_TIMESTAMP_DATA_TYPEY + USE_POOLINGN + + + + diff --git a/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr b/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr new file mode 100644 index 0000000..f932bd1 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/DefineSqlVariables.ktr @@ -0,0 +1,749 @@ + + + + DefineSqlVariables + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/24 15:35:39.260 + - + 2016/10/25 12:50:14.859 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + ReadMetadataFile + DefineSqlVariablesWorker + Y + + + + DefineSqlVariablesWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/DefineSqlVariablesWorker.ktr + + ReadCsv + + + N + + + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + + + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + + + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + + + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + + + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + + + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + + + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + + + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + + + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + + + + + + + + + 288 + 64 + Y + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + + + + + + + 96 + 64 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr b/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr new file mode 100644 index 0000000..bd37da9 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/DefineSqlVariablesWorker.ktr @@ -0,0 +1,716 @@ + + + + DefineSqlVariablesWorker + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 12:43:03.614 + - + 2016/10/26 11:42:46.247 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + DefineSqlVariables + SetVariables + Y + + + ReadCsv + FirstRowOnly + Y + + + FirstRowOnly + DefineSqlVariables + Y + + + + DefineSqlVariables + UserDefinedJavaClass + + Y + + 1 + + none + + + + + TRANSFORM_CLASS + Processor + import org.pentaho.di.core.database.*; +import java.util.*; + +private boolean first = true; + +public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException +{ + Object[] row = this.getRow(); + + if (this.first) + { + this.first = false; + + String schemaName = this.getVariable("OUTPUT_SCHEMA", ""); + + + String tableName = this.getVariable("OUTPUT_TABLE", ""); + + RowMetaInterface inputRowMeta = this.getInputRowMeta(); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!1"); + + DatabaseMeta databaseMeta = this.getStepMeta().getParentTransMeta().findDatabase("Library_ExportCsvToTable_Connection"); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!2"); + logBasic(String.valueOf(databaseMeta)); + DatabaseMeta databaseMeta3 = this.getTransMeta().findDatabase("Library_ExportCsvToTable_Connection"); + logBasic("!!!!!!!!!!!!!!!!!!!!!!!3"); + logBasic(String.valueOf(databaseMeta3)); + + Database database = new Database(this.getTrans(), databaseMeta3); + + String schemaTable = databaseMeta.getQuotedSchemaTableCombination(schemaName, tableName); + + String sqlCreate = database.getDDLCreationTable(schemaTable, inputRowMeta); + + String sqlDrop = "DROP TABLE " + schemaTable + ";"; + + Object[] outputRow = createOutputRow(row, this.data.outputRowMeta.size()); + + this.get(Fields.Out, "SqlCreate").setValue(outputRow, sqlCreate); + + this.get(Fields.Out, "SqlDrop").setValue(outputRow, sqlDrop); + + this.putRow(this.data.outputRowMeta, outputRow); + + return true; + } + + if (row != null) + { + return true; + } + else + { + this.setOutputDone(); + + return false; + } +} + + + + + SqlCreate + String + -1 + -1 + + + SqlDrop + String + -1 + -1 + + + Y + + + + + + + + + + + + 416 + 96 + Y + + + + FirstRowOnly + SampleRows + + Y + + 1 + + none + + + 1 + + + + + + + + + + 272 + 96 + Y + + + + ReadCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + + N + ${INPUT_DELIMITER} + ${INPUT_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + + + + + + + + + 128 + 96 + Y + + + + SetVariables + SetVariable + + Y + + 1 + + none + + + + + SqlCreate + SQL_CREATE + PARENT_JOB + + + + SqlDrop + SQL_DROP + PARENT_JOB + + + + Y + + + + + + + + + 624 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb b/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb new file mode 100644 index 0000000..103eb62 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvInDirectoryToDatabase.kjb @@ -0,0 +1,566 @@ + + + ExportCsvInDirectoryToDatabase + Exports all CSV files in a directory to homonymous tables in a database. + Exports all CSV files in a directory to homonymous tables in a database. + +Dependencies: +- /Library/ExportCsvToTable (folder) + + + 0 + / + - + 2016/10/25 13:40:04.860 + - + 2016/10/25 15:12:50.741 + + + INPUT_DELIMITER + , + The delimiter character in the CSV files in the input directory. + + + INPUT_DIRECTORY + + The input directory. + + + INPUT_ENCLOSURE + " + The enclosure character in the CSV files in the input directory. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output tables in. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 96 + 96 + + + GetCsvInDirectoryInformation + + TRANS + filename + + ${Internal.Entry.Current.Directory}/GetCsvInDirectoryInformation.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + Y + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_DIRECTORY + + ${INPUT_DIRECTORY} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + N + Y + 0 + 256 + 96 + + + ExportCsvToTable + + JOB + filename + + ${Internal.Entry.Current.Directory}/ExportCsvToTable.kjb + + N + Y + Y + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + FilePath + + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + TableName + ${OUTPUT_TABLE} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + N + Y + 0 + 512 + 96 + + + Success + + SUCCESS + N + Y + 0 + 688 + 96 + + + + + START + GetCsvInDirectoryInformation + 0 + 0 + Y + Y + Y + + + ExportCsvToTable + Success + 0 + 0 + Y + Y + N + + + GetCsvInDirectoryInformation + ExportCsvToTable + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb new file mode 100644 index 0000000..157e198 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.kjb @@ -0,0 +1,887 @@ + + + ExportCsvToTable + Exports a CSV file to a database table. + Exports a CSV file to a database table. + +Dependencies: +- /Library/ExtractCsvMetadataToCsv (folder) +- /Library_ExtractCsvMetadataToCsv_Connection (connection) + + 0 + / + - + 2016/10/23 21:53:12.195 + - + 2016/11/03 00:00:44.629 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DRIVER + + The name of the driver class used to connect to the output database. + + + OUTPUT_PASSWORD + + The password used to connect to the output database. + + + OUTPUT_SCHEMA + + The name of the schema to store the output table in. + + + OUTPUT_TABLE + + The name of the output table. + + + OUTPUT_URL + + The url of the output database. + + + OUTPUT_USERNAME + + The username used to connect to the output database. + + + + Library_ExportCsvToTable_Connection + + GENERIC + Native + + 1521 + ${OUTPUT_USERNAME} + ${OUTPUT_PASSWORD} + + + + + + CUSTOM_DRIVER_CLASS + ${OUTPUT_DRIVER} + + + CUSTOM_URL + ${OUTPUT_URL} + + + FORCE_IDENTIFIERS_TO_LOWERCASE + N + + + FORCE_IDENTIFIERS_TO_UPPERCASE + N + + + IS_CLUSTERED + N + + + PORT_NUMBER + 1521 + + + PRESERVE_RESERVED_WORD_CASE + Y + + + QUOTE_ALL_FIELDS + N + + + SUPPORTS_BOOLEAN_DATA_TYPE + Y + + + SUPPORTS_TIMESTAMP_DATA_TYPE + Y + + + USE_POOLING + N + + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 64 + + + ExtractCsvMetadataToCsv + + JOB + filename + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsv.kjb + + N + N + N + N + + + N + N + Nothing + + Y + N + N + N + N + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + OUTPUT_PATH + + ${METADATA_PATH} + + + OUTPUT_DELIMITER + + ${METADATA_DELIMITER} + + + OUTPUT_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + N + N + Y + 0 + 480 + 64 + + + DefineMetadataVariables + + EVAL + + N + Y + 0 + 256 + 64 + + + ExportCsvToTable + + TRANS + filename + + ${Internal.Entry.Current.Directory}/ExportCsvToTable.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + METADATA_DELIMITER + + ${METADATA_DELIMITER} + + + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + METADATA_PATH + + ${METADATA_PATH} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + + ${OUTPUT_TABLE} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + N + Y + 0 + 544 + 240 + + + DeleteMetadataFile + + DELETE_FILE + ${METADATA_PATH} + N + N + Y + 0 + 704 + 240 + + + DefineSqlVariables + + TRANS + filename + + ${Internal.Entry.Current.Directory}/DefineSqlVariables.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + METADATA_DELIMITER + + ${METADATA_DELIMITER} + + + METADATA_ENCLOSURE + + ${METADATA_ENCLOSURE} + + + METADATA_PATH + + ${METADATA_PATH} + + + OUTPUT_DRIVER + + ${OUTPUT_DRIVER} + + + OUTPUT_PASSWORD + + ${OUTPUT_PASSWORD} + + + OUTPUT_SCHEMA + + ${OUTPUT_SCHEMA} + + + OUTPUT_TABLE + + ${OUTPUT_TABLE} + + + OUTPUT_URL + + ${OUTPUT_URL} + + + OUTPUT_USERNAME + + ${OUTPUT_USERNAME} + + + N + Y + 0 + 720 + 64 + + + TableExists + + TABLE_EXISTS + ${OUTPUT_TABLE} + ${OUTPUT_SCHEMA} + Library_ExportCsvToTable_Connection + N + Y + 0 + 80 + 240 + + + DropTable + + SQL + ${SQL_DROP} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 240 + 304 + + + CreateTable + + SQL + ${SQL_CREATE} + T + F + + T + Library_ExportCsvToTable_Connection + N + Y + 0 + 384 + 240 + + + Success + + SUCCESS + N + Y + 0 + 848 + 240 + + + + + ExportCsvToTable + DeleteMetadataFile + 0 + 0 + Y + Y + N + + + START + DefineMetadataVariables + 0 + 0 + Y + Y + Y + + + DefineMetadataVariables + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + N + + + ExtractCsvMetadataToCsv + DefineSqlVariables + 0 + 0 + Y + Y + N + + + DefineSqlVariables + TableExists + 0 + 0 + Y + Y + N + + + TableExists + DropTable + 0 + 0 + Y + Y + N + + + TableExists + CreateTable + 0 + 0 + Y + N + N + + + DropTable + CreateTable + 0 + 0 + Y + Y + N + + + CreateTable + ExportCsvToTable + 0 + 0 + Y + Y + N + + + DeleteMetadataFile + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr new file mode 100644 index 0000000..d350115 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExportCsvToTable.ktr @@ -0,0 +1,749 @@ + + + + ExportCsvToTable + + + + Normal + 0 + / + + + INPUT_DELIMITER + + + + + INPUT_ENCLOSURE + + + + + INPUT_PATH + + + + + METADATA_DELIMITER + + + + + METADATA_ENCLOSURE + + + + + METADATA_PATH + + + + + OUTPUT_DRIVER + + + + + OUTPUT_PASSWORD + + + + + OUTPUT_SCHEMA + + + + + OUTPUT_TABLE + + + + + OUTPUT_URL + + + + + OUTPUT_USERNAME + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/23 18:35:54.307 + - + 2016/10/25 12:29:25.654 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + ReadMetadataFile + ExportCsvToTableWorker + Y + + + + ExportCsvToTableWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/ExportCsvToTableWorker.ktr + + ReadCsv + + + N + + + + + ReadCsv + FIELD_LENGTH + Y + ReadMetadataFile + Length + + + ReadCsv + FIELD_TRIM_TYPE + Y + ReadMetadataFile + TrimType + + + ReadCsv + FIELD_DECIMAL + Y + ReadMetadataFile + Decimal + + + ReadCsv + FIELD_NAME + Y + ReadMetadataFile + Name + + + ReadCsv + FIELD_FORMAT + Y + ReadMetadataFile + Format + + + ReadCsv + FIELD_PRECISION + Y + ReadMetadataFile + Precision + + + ReadCsv + FIELD_CURRENCY + Y + ReadMetadataFile + Currency + + + ReadCsv + FIELD_TYPE + Y + ReadMetadataFile + Type + + + ReadCsv + FIELD_GROUP + Y + ReadMetadataFile + Group + + + + + + + + + + + 352 + 96 + Y + + + + ReadMetadataFile + CsvInput + + Y + + 1 + + none + + + ${METADATA_PATH} + + + N + ${METADATA_DELIMITER} + ${METADATA_ENCLOSURE} +
Y
+ 50000 + Y + N + N + N + + + + Name + String + + + + + -1 + -1 + none + + + Type + String + + + + + -1 + -1 + none + + + Format + String + + + + + -1 + -1 + none + + + Length + String + + + + + -1 + -1 + none + + + Precision + String + + + + + -1 + -1 + none + + + Currency + String + + + + + -1 + -1 + none + + + Decimal + String + + + + + -1 + -1 + none + + + Group + String + + + + + -1 + -1 + none + + + TrimType + String + + + + + -1 + -1 + none + + + + + + + + + + + 144 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr b/pentaho/kissflow/BlocPowerETL/ExportCsvToTableWorker.ktr similarity index 100% rename from pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTableWorker.ktr rename to pentaho/kissflow/BlocPowerETL/ExportCsvToTableWorker.ktr diff --git a/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb new file mode 100644 index 0000000..4eabedc --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.kjb @@ -0,0 +1,460 @@ + + + ExtractCsvMetadataToCsv + Extracts metadata from a CSV file and saves it to another CSV file. + Extracts metadata from a CSV file and saves it to another CSV file. + +This job assumes that the input CSV file has at least two rows: +- The first is the header row (from which the names of the fields are taken). +- The second is a data row (from which the types of the fields are inferred). + + + 0 + / + - + 2016/10/22 16:22:05.789 + - + 2016/11/02 23:00:11.519 + + + INPUT_DELIMITER + , + The delimiter character in the input CSV file. + + + INPUT_ENCLOSURE + " + The enclosure character in the input CSV file. + + + INPUT_PATH + + The path to the input CSV file. + + + OUTPUT_DELIMITER + , + The delimiter character in the output metadata (CSV) file. + + + OUTPUT_ENCLOSURE + " + The delimiter character in the enclosure metadata (CSV) file. + + + OUTPUT_PATH + + The path of the new metadata (CSV) file. + + + + + + + +
+ + + + + ID_JOB + Y + ID_JOB + + + CHANNEL_ID + Y + CHANNEL_ID + + + JOBNAME + Y + JOBNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + START_JOB_ENTRY + N + START_JOB_ENTRY + + + CLIENT + N + CLIENT + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + JOBNAME + Y + TRANSNAME + + + JOBENTRYNAME + Y + STEPNAME + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + RESULT + Y + RESULT + + + NR_RESULT_ROWS + Y + NR_RESULT_ROWS + + + NR_RESULT_FILES + Y + NR_RESULT_FILES + + + LOG_FIELD + N + LOG_FIELD + + + COPY_NR + N + COPY_NR + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + N + + + + START + + SPECIAL + Y + N + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + Y + 0 + 80 + 48 + + + ExtractCsvMetadataToCsv + + TRANS + filename + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsv.ktr + + N + N + N + N + N + N + + + N + N + Basic + N + + N + Y + N + N + N + Pentaho local + + N + + INPUT_DELIMITER + + ${INPUT_DELIMITER} + + + INPUT_ENCLOSURE + + ${INPUT_ENCLOSURE} + + + INPUT_PATH + + ${INPUT_PATH} + + + OUTPUT_DELIMITER + + ${OUTPUT_DELIMITER} + + + OUTPUT_ENCLOSURE + + ${OUTPUT_ENCLOSURE} + + + OUTPUT_PATH + + ${OUTPUT_PATH} + + + N + Y + 0 + 256 + 48 + + + Success + + SUCCESS + N + Y + 0 + 432 + 48 + + + + + START + ExtractCsvMetadataToCsv + 0 + 0 + Y + Y + Y + + + ExtractCsvMetadataToCsv + Success + 0 + 0 + Y + Y + N + + + + + + + METASTORE.pentaho + + Default Run Configuration + {"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null} + + + + {"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"} + + Pentaho local + {"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]} + + + + diff --git a/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr new file mode 100644 index 0000000..be1b618 --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsv.ktr @@ -0,0 +1,1116 @@ + + + + ExtractCsvMetadataToCsv + + + + Normal + 0 + / + + + INPUT_DELIMITER + , + + + + INPUT_ENCLOSURE + " + + + + INPUT_PATH + + + + + OUTPUT_DELIMITER + , + + + + OUTPUT_ENCLOSURE + " + + + + OUTPUT_PATH + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/22 13:07:22.373 + - + 2016/11/03 14:01:30.830 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + KeepHeaderOnly + BuildRowForEachField + Y + + + BuildRowForEachField + CleanRowSet + Y + + + CleanRowSet + PickFieldName + Y + + + PickFieldName + AddFieldTypeAsString + Y + + + AddFieldTypeAsString + ExtractCsvMetadataToCsvWorker + Y + + + ExtractCsvMetadataToCsvWorker + WriteCsv + Y + + + ReadLinesFromCsv + KeepHeaderOnly + Y + + + + AddFieldTypeAsString + Constant + + Y + + 1 + + none + + + + + FieldType + String + + + + + String + -1 + -1 + N + + + + + + + + + + + 240 + 224 + Y + + + + BuildRowForEachField + ScriptValueMod + + Y + + 1 + + none + + + N + 9 + + + 1 + ScriptStart + +// DEFAULT VALUES: + +var FieldName = null; + +// AUXILIARY FUNCTIONS: + +function SplitLine(line) +{ + var delimiter = getVariable("INPUT_DELIMITER",","); + var enclosure = getVariable("INPUT_ENCLOSURE", "\""); + var fields = []; + var currentField = ""; + var enclosureSeen = false; + + for (var i = 0; i < line.length; i++) + { + var character = line.charAt(i); + + if (enclosureSeen) + { + if (character == enclosure) + { + if (i < line.length - 1 && line.charAt(i + 1) == enclosure) + { + currentField += enclosure; + i += 1; + } + else + { + enclosureSeen = false; + } + } + else + { + currentField += character; + } + } + else + { + if (character == delimiter) + { + fields.push(currentField); + currentField = ""; + } + else if (character == enclosure) + { + enclosureSeen = true; + } + else + { + currentField += character; + } + } + } + + fields.push(currentField); + currentField = ""; + enclosureSeen = false; + + var regexp = new RegExp(enclosure, "g"); + + for (var i = 0; i < fields.length; i++) + { + fields[i] = fields[i].trim(); + + if (enclosure == "\"") + { + fields[i] = fields[i].replace(regexp, "'"); + } + } + + return fields; +} + + + 0 + ScriptLoop + +// LOOP CODE: + +var fields = SplitLine(LineText); + +for (var i = 0; i < fields.length; i++) +{ + var newRow = createRowCopy(getOutputRowMeta().size()); + + var rowIndex = getInputRowMeta().size(); + + // FieldName + newRow[rowIndex] = fields[i]; + + putRow(newRow); +} + + + + + + + + FieldName + FieldName + String + -1 + -1 + N + + + + + + + + + + + 416 + 64 + Y + + + + CleanRowSet + FilterRows + + Y + + 1 + + none + + + PickFieldName + + + + N + FieldName + IS NOT NULL + + + + + + + + + + + + 624 + 64 + Y + + + + ExtractCsvMetadataToCsvWorker + MetaInject + + Y + + 1 + + none + + + filename + + + ${Internal.Entry.Current.Directory}/ExtractCsvMetadataToCsvWorker.ktr + + PickMetadataFields + + + Name + String + -1 + -1 + + + Type + String + -1 + -1 + + + Format + String + -1 + -1 + + + Length + String + -1 + -1 + + + Precision + String + -1 + -1 + + + Currency + String + -1 + -1 + + + Decimal + String + -1 + -1 + + + Group + String + -1 + -1 + + + TrimType + String + -1 + -1 + + + + N + + + + + ReadCsv + FIELD_TYPE + Y + AddFieldTypeAsString + FieldType + + + ReadCsv + FIELD_NAME + Y + AddFieldTypeAsString + FieldName + + + + + + + + + + + 448 + 224 + Y + + + + KeepHeaderOnly + FilterRows + + Y + + 1 + + none + + + BuildRowForEachField + + + + N + LineNumber + = + + + constant + Integer + 1 + -1 + 0 + N + #;-# + + + + + + + + + + + + 240 + 64 + Y + + + + PickFieldName + SelectValues + + Y + + 1 + + none + + + + + FieldName + + N + + + + + + + + + + 80 + 224 + Y + + + + ReadLinesFromCsv + CsvInput + + Y + + 1 + + none + + + ${INPUT_PATH} + + LineNumber + N + $[00] + $[01] +
N
+ 50000 + Y + N + N + N + + + + LineText + String + + + + + -1 + -1 + none + + + + + + + + + + + 80 + 64 + Y + + + + WriteCsv + TextFileOutput + + Y + + 1 + + none + + + ${OUTPUT_DELIMITER} + ${OUTPUT_ENCLOSURE} + N + N +
Y
+
N
+ DOS + None + windows-1252 + + N + + Y + + ${OUTPUT_PATH} + N + N + N + + N + N + N + N + N + N + + Y + N + N + 0 + + + + Name + String + + + + + + none + -1 + -1 + + + Type + String + + + + + + none + -1 + -1 + + + Format + String + + + + + + none + -1 + -1 + + + Length + String + + + + + + none + -1 + -1 + + + Precision + String + + + + + + none + -1 + -1 + + + Currency + String + + + + + + none + -1 + -1 + + + Decimal + String + + + + + + none + -1 + -1 + + + Group + String + + + + + + none + -1 + -1 + + + TrimType + String + + + + + + none + -1 + -1 + + + + + + + + + + + 608 + 224 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr b/pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsvWorker.ktr similarity index 100% rename from pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsvWorker.ktr rename to pentaho/kissflow/BlocPowerETL/ExtractCsvMetadataToCsvWorker.ktr diff --git a/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr b/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr new file mode 100644 index 0000000..3f3accb --- /dev/null +++ b/pentaho/kissflow/BlocPowerETL/GetCsvInDirectoryInformation.ktr @@ -0,0 +1,645 @@ + + + + GetCsvInDirectoryInformation + + + + Normal + 0 + / + + + INPUT_DIRECTORY + + + + + + + + +
+ + + + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + TRANSNAME + Y + TRANSNAME + + + STATUS + Y + STATUS + + + LINES_READ + Y + LINES_READ + + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + + LINES_UPDATED + Y + LINES_UPDATED + + + + LINES_INPUT + Y + LINES_INPUT + + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + + LINES_REJECTED + Y + LINES_REJECTED + + + + ERRORS + Y + ERRORS + + + STARTDATE + Y + STARTDATE + + + ENDDATE + Y + ENDDATE + + + LOGDATE + Y + LOGDATE + + + DEPDATE + Y + DEPDATE + + + REPLAYDATE + Y + REPLAYDATE + + + LOG_FIELD + Y + LOG_FIELD + + + EXECUTING_SERVER + N + EXECUTING_SERVER + + + EXECUTING_USER + N + EXECUTING_USER + + + CLIENT + N + CLIENT + + + + + +
+ + + + ID_BATCH + Y + ID_BATCH + + + SEQ_NR + Y + SEQ_NR + + + LOGDATE + Y + LOGDATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + INPUT_BUFFER_ROWS + Y + INPUT_BUFFER_ROWS + + + OUTPUT_BUFFER_ROWS + Y + OUTPUT_BUFFER_ROWS + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + LOGGING_OBJECT_TYPE + Y + LOGGING_OBJECT_TYPE + + + OBJECT_NAME + Y + OBJECT_NAME + + + OBJECT_COPY + Y + OBJECT_COPY + + + REPOSITORY_DIRECTORY + Y + REPOSITORY_DIRECTORY + + + FILENAME + Y + FILENAME + + + OBJECT_ID + Y + OBJECT_ID + + + OBJECT_REVISION + Y + OBJECT_REVISION + + + PARENT_CHANNEL_ID + Y + PARENT_CHANNEL_ID + + + ROOT_CHANNEL_ID + Y + ROOT_CHANNEL_ID + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + TRANSNAME + Y + TRANSNAME + + + STEPNAME + Y + STEPNAME + + + STEP_COPY + Y + STEP_COPY + + + LINES_READ + Y + LINES_READ + + + LINES_WRITTEN + Y + LINES_WRITTEN + + + LINES_UPDATED + Y + LINES_UPDATED + + + LINES_INPUT + Y + LINES_INPUT + + + LINES_OUTPUT + Y + LINES_OUTPUT + + + LINES_REJECTED + Y + LINES_REJECTED + + + ERRORS + Y + ERRORS + + + LOG_FIELD + N + LOG_FIELD + + + + + +
+ + + ID_BATCH + Y + ID_BATCH + + + CHANNEL_ID + Y + CHANNEL_ID + + + LOG_DATE + Y + LOG_DATE + + + METRICS_DATE + Y + METRICS_DATE + + + METRICS_CODE + Y + METRICS_CODE + + + METRICS_DESCRIPTION + Y + METRICS_DESCRIPTION + + + METRICS_SUBJECT + Y + METRICS_SUBJECT + + + METRICS_TYPE + Y + METRICS_TYPE + + + METRICS_VALUE + Y + METRICS_VALUE + + + + + +
+ + 0.0 + 0.0 + + 10000 + 50 + 50 + N + Y + 50000 + Y + + N + 1000 + 100 + + + + + + + + + - + 2016/10/25 13:44:34.382 + - + 2016/10/25 14:47:13.667 + H4sIAAAAAAAAAAMAAAAAAAAAAAA= + N + + + + + + GetCsvNames + SelectFields + Y + + + SelectFields + TableName + Y + + + TableName + SelectFields2 + Y + + + SelectFields2 + CopyRowsToResult + Y + + + + CopyRowsToResult + RowsToResult + + Y + + 1 + + none + + + + + + + + + + + 608 + 96 + Y + + + + GetCsvNames + GetFileNames + + Y + + 1 + + none + + + + only_files + + N + N + Y + N + + + + + N + 0 + + ${INPUT_DIRECTORY} + .*\.csv$ + + Y + N + + + + N + N + + + + + + + + + + 80 + 96 + Y + + + + SelectFields + SelectValues + + Y + + 1 + + none + + + + + filename + FilePath + + + short_filename + FileName + + N + + + + + + + + + + 208 + 96 + Y + + + + SelectFields2 + SelectValues + + Y + + 1 + + none + + + + + FilePath + + + TableName + + N + + + + + + + + + + 464 + 96 + Y + + + + TableName + Formula + + Y + + 1 + + none + + + + TableName + LEFT([FileName]; LEN([FileName])-4) + + String + -1 + -1 + + + + + + + + + + + 336 + 96 + Y + + + + + + + N + diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb deleted file mode 100644 index e8e3af8..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/ExportCsvInDirectoryToDatabase.kjb +++ /dev/null @@ -1,256 +0,0 @@ - - ExportCsvInDirectoryToDatabase - Exports all CSV files in a directory to homonymous tables in a database. - Exports all CSV files in a directory to homonymous tables in a database. Dependencies: - /Library/ExportCsvToTable (folder) - - 0 - /Library/ExportCsvInDirectoryToDatabase - - - 2016/10/25 13:40:04.860 - - - 2016/10/25 15:12:50.741 - - - INPUT_DELIMITER - , - The delimiter character in the CSV files in the input directory. - - - INPUT_DIRECTORY - - The input directory. - - - INPUT_ENCLOSURE - " - The enclosure character in the CSV files in the input directory. - - - OUTPUT_DRIVER - - The name of the driver class used to connect to the output database. - - - OUTPUT_PASSWORD - - The password used to connect to the output database. - - - OUTPUT_SCHEMA - - The name of the schema to store the output tables in. - - - OUTPUT_URL - - The url of the output database. - - - OUTPUT_USERNAME - - The username used to connect to the output database. - - - - - - -
- - - -ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - N - - - - START - - SPECIAL - Y - N - N - 0 - 0 - 60 - 12 - 0 - 1 - 1 - N - Y - 0 - 96 - 96 - - - GetCsvInDirectoryInformation - - TRANS - rep_name - - - GetCsvInDirectoryInformation - /Library/ExportCsvInDirectoryToDatabase - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_DIRECTORY - - ${INPUT_DIRECTORY} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - OUTPUT_DRIVER - - ${OUTPUT_DRIVER} - OUTPUT_PASSWORD - - ${OUTPUT_PASSWORD} - OUTPUT_SCHEMA - - ${OUTPUT_SCHEMA} - OUTPUT_URL - - ${OUTPUT_URL} - OUTPUT_USERNAME - - ${OUTPUT_USERNAME} - N - Y - 0 - 240 - 96 - - - ExportCsvToTable - - JOB - rep_name - - - ExportCsvToTable - /Library/ExportCsvToTable - N - Y - Y - N - - - N - N - Nothing - - Y - N - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - INPUT_PATH - FilePath - - OUTPUT_DRIVER - - ${OUTPUT_DRIVER} - OUTPUT_PASSWORD - - ${OUTPUT_PASSWORD} - OUTPUT_SCHEMA - - ${OUTPUT_SCHEMA} - OUTPUT_TABLE - TableName - ${OUTPUT_TABLE} - OUTPUT_URL - - ${OUTPUT_URL} - OUTPUT_USERNAME - - ${OUTPUT_USERNAME} - N - N - Y - 0 - 416 - 96 - - - Success - - SUCCESS - N - Y - 0 - 576 - 96 - - - - - START - GetCsvInDirectoryInformation - 0 - 0 - Y - Y - Y - - - GetCsvInDirectoryInformation - ExportCsvToTable - 0 - 0 - Y - Y - N - - - ExportCsvToTable - Success - 0 - 0 - Y - Y - N - - - - - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr deleted file mode 100644 index e0bef1e..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvInDirectoryToDatabase/GetCsvInDirectoryInformation.ktr +++ /dev/null @@ -1,239 +0,0 @@ - - - GetCsvInDirectoryInformation - - - - Normal - 0 - /Library/ExportCsvInDirectoryToDatabase - - - INPUT_DIRECTORY - - - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT - - -
- - -ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2016/10/25 13:44:34.382 - - - 2016/10/25 14:47:13.667 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - GetCsvNamesSelectFieldsY - SelectFieldsTableNameY - TableNameSelectFields2Y - SelectFields2CopyRowsToResultY - - - CopyRowsToResult - RowsToResult - - Y - - 1 - - none - - - - - 608 - 96 - Y - - - - - GetCsvNames - GetFileNames - - Y - - 1 - - none - - - - only_files - - N - N - Y - N - - - - - N - 0 - - ${INPUT_DIRECTORY} - .*\.csv$ - - N - N - - - - N - N - - - - 80 - 96 - Y - - - - - SelectFields - SelectValues - - Y - - 1 - - none - - - filename - FilePath - -2 - -2 - short_filename - FileName - -2 - -2 - N - - - 208 - 96 - Y - - - - - SelectFields2 - SelectValues - - Y - - 1 - - none - - - FilePath - - -2 - -2 - TableName - - -2 - -2 - N - - - 464 - 96 - Y - - - - - TableName - Formula - - Y - - 1 - - none - - - TableName -LEFT([FileName]; LEN([FileName])-4) -String --1 --1 - - - - - 336 - 96 - Y - - - - - - - - N - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr deleted file mode 100644 index eba0c55..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariables.ktr +++ /dev/null @@ -1,353 +0,0 @@ - - - DefineSqlVariables - - - - Normal - 0 - /Library/ExportCsvToTable - - - INPUT_DELIMITER - - - - - INPUT_ENCLOSURE - - - - - INPUT_PATH - - - - - METADATA_DELIMITER - - - - - METADATA_ENCLOSURE - - - - - METADATA_PATH - - - - - OUTPUT_DRIVER - - - - - OUTPUT_PASSWORD - - - - - OUTPUT_SCHEMA - - - - - OUTPUT_TABLE - - - - - OUTPUT_URL - - - - - OUTPUT_USERNAME - - - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT - - -
- - -ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2016/10/24 15:35:39.260 - - - 2016/10/25 12:50:14.859 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - ReadMetadataFileDefineSqlVariablesWorkerY - - - DefineSqlVariablesWorker - MetaInject - - Y - - 1 - - none - - - rep_name - - DefineSqlVariablesWorker - - /Library/ExportCsvToTable - ReadCsv - - N - - - ReadCsv - FIELD_LENGTH - Y - ReadMetadataFile - Length - ReadCsv - FIELD_TRIM_TYPE - Y - ReadMetadataFile - TrimType - ReadCsv - FIELD_DECIMAL - Y - ReadMetadataFile - Decimal - ReadCsv - FIELD_NAME - Y - ReadMetadataFile - Name - ReadCsv - FIELD_FORMAT - Y - ReadMetadataFile - Format - ReadCsv - FIELD_PRECISION - Y - ReadMetadataFile - Precision - ReadCsv - FIELD_CURRENCY - Y - ReadMetadataFile - Currency - ReadCsv - FIELD_TYPE - Y - ReadMetadataFile - Type - ReadCsv - FIELD_GROUP - Y - ReadMetadataFile - Group - - - 288 - 64 - Y - - - - - ReadMetadataFile - CsvInput - - Y - - 1 - - none - - - ${METADATA_PATH} - - - N - ${METADATA_DELIMITER} - ${METADATA_ENCLOSURE} -
Y
- 50000 - Y - N - N - N - - - - Name - String - - - - - -1 - -1 - none - - - Type - String - - - - - -1 - -1 - none - - - Format - String - - - - - -1 - -1 - none - - - Length - String - - - - - -1 - -1 - none - - - Precision - String - - - - - -1 - -1 - none - - - Currency - String - - - - - -1 - -1 - none - - - Decimal - String - - - - - -1 - -1 - none - - - Group - String - - - - - -1 - -1 - none - - - TrimType - String - - - - - -1 - -1 - none - - - - - 112 - 64 - Y - - - - - - - - N - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr deleted file mode 100644 index 9de0340..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/DefineSqlVariablesWorker.ktr +++ /dev/null @@ -1,323 +0,0 @@ - - - DefineSqlVariablesWorker - - - - Normal - 0 - /Library/ExportCsvToTable - - - INPUT_DELIMITER - - - - - INPUT_ENCLOSURE - - - - - INPUT_PATH - - - - - OUTPUT_DRIVER - - - - - OUTPUT_PASSWORD - - - - - OUTPUT_SCHEMA - - - - - OUTPUT_TABLE - - - - - OUTPUT_URL - - - - - OUTPUT_USERNAME - - - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT - - -
- - -ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2016/10/25 12:43:03.614 - - - 2016/10/26 11:42:46.247 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - DefineSqlVariablesSetVariablesY - ReadCsvFirstRowOnlyY - FirstRowOnlyDefineSqlVariablesY - - - DefineSqlVariables - UserDefinedJavaClass - - Y - - 1 - - none - - - - - - TRANSFORM_CLASS - - Processor - - - - - - - SqlCreate - - String - - -1 - - -1 - - - - SqlDrop - - String - - -1 - - -1 - - - Y - - - 432 - 96 - Y - - - - - ReadCsv - CsvInput - - Y - - 1 - - none - - - ${INPUT_PATH} - - - N - ${INPUT_DELIMITER} - ${INPUT_ENCLOSURE} -
Y
- 50000 - Y - N - N - N - - - - - - 128 - 96 - Y - - - - - SetVariables - SetVariable - - Y - - 1 - - none - - - - - SqlCreate - SQL_CREATE - PARENT_JOB - - - - SqlDrop - SQL_DROP - PARENT_JOB - - - - Y - - - 624 - 96 - Y - - - - - FirstRowOnly - SampleRows - - Y - - 1 - - none - - - 1 - - - - 272 - 96 - Y - - - - - - - - N - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb deleted file mode 100644 index a3d577a..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.kjb +++ /dev/null @@ -1,491 +0,0 @@ - - ExportCsvToTable - Exports a CSV file to a database table. - Exports a CSV file to a database table. Dependencies: - /Library/ExtractCsvMetadataToCsv (folder) - /Library_ExtractCsvMetadataToCsv_Connection (connection) - - 0 - /Library/ExportCsvToTable - - - 2016/10/23 21:53:12.195 - - - 2016/11/03 00:00:44.629 - - - INPUT_DELIMITER - , - The delimiter character in the input CSV file. - - - INPUT_ENCLOSURE - " - The enclosure character in the input CSV file. - - - INPUT_PATH - - The path to the input CSV file. - - - OUTPUT_DRIVER - - The name of the driver class used to connect to the output database. - - - OUTPUT_PASSWORD - - The password used to connect to the output database. - - - OUTPUT_SCHEMA - - The name of the schema to store the output table in. - - - OUTPUT_TABLE - - The name of the output table. - - - OUTPUT_URL - - The url of the output database. - - - OUTPUT_USERNAME - - The username used to connect to the output database. - - - - Library_ExportCsvToTable_Connection - - GENERIC - Native - - 1521 - ${OUTPUT_USERNAME} - ${OUTPUT_PASSWORD} - - - - - CUSTOM_DRIVER_CLASS${OUTPUT_DRIVER} - CUSTOM_URL${OUTPUT_URL} - FORCE_IDENTIFIERS_TO_LOWERCASEN - FORCE_IDENTIFIERS_TO_UPPERCASEN - IS_CLUSTEREDN - PORT_NUMBER1521 - PRESERVE_RESERVED_WORD_CASEY - QUOTE_ALL_FIELDSY - SUPPORTS_BOOLEAN_DATA_TYPEY - SUPPORTS_TIMESTAMP_DATA_TYPEY - USE_POOLINGN - - - - - - -
- - - -ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - N - - - - START - - SPECIAL - Y - N - N - 0 - 0 - 60 - 12 - 0 - 1 - 1 - N - Y - 0 - 80 - 64 - - - ExtractCsvMetadataToCsv - - JOB - rep_name - - - ExtractCsvMetadataToCsv - /Library/ExtractCsvMetadataToCsv - N - N - N - N - - - N - N - Nothing - - Y - N - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - INPUT_PATH - - ${INPUT_PATH} - OUTPUT_PATH - - ${METADATA_PATH} - OUTPUT_DELIMITER - - ${METADATA_DELIMITER} - OUTPUT_ENCLOSURE - - ${METADATA_ENCLOSURE} - N - N - Y - 0 - 480 - 64 - - - DefineMetadataVariables - - EVAL - - N - Y - 0 - 256 - 64 - - - ExportCsvToTable - - TRANS - rep_name - - - ExportCsvToTable - /Library/ExportCsvToTable - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - INPUT_PATH - - ${INPUT_PATH} - METADATA_DELIMITER - - ${METADATA_DELIMITER} - METADATA_ENCLOSURE - - ${METADATA_ENCLOSURE} - METADATA_PATH - - ${METADATA_PATH} - OUTPUT_URL - - ${OUTPUT_URL} - OUTPUT_SCHEMA - - ${OUTPUT_SCHEMA} - OUTPUT_TABLE - - ${OUTPUT_TABLE} - OUTPUT_DRIVER - - ${OUTPUT_DRIVER} - OUTPUT_USERNAME - - ${OUTPUT_USERNAME} - OUTPUT_PASSWORD - - ${OUTPUT_PASSWORD} - N - Y - 0 - 544 - 240 - - - DeleteMetadataFile - - DELETE_FILE - ${METADATA_PATH} - N - N - Y - 0 - 704 - 240 - - - DefineSqlVariables - - TRANS - rep_name - - - DefineSqlVariables - /Library/ExportCsvToTable - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - INPUT_PATH - - ${INPUT_PATH} - METADATA_DELIMITER - - ${METADATA_DELIMITER} - METADATA_ENCLOSURE - - ${METADATA_ENCLOSURE} - METADATA_PATH - - ${METADATA_PATH} - OUTPUT_DRIVER - - ${OUTPUT_DRIVER} - OUTPUT_PASSWORD - - ${OUTPUT_PASSWORD} - OUTPUT_SCHEMA - - ${OUTPUT_SCHEMA} - OUTPUT_TABLE - - ${OUTPUT_TABLE} - OUTPUT_URL - - ${OUTPUT_URL} - OUTPUT_USERNAME - - ${OUTPUT_USERNAME} - N - Y - 0 - 720 - 64 - - - TableExists - - TABLE_EXISTS - ${OUTPUT_TABLE} - ${OUTPUT_SCHEMA} - Library_ExportCsvToTable_Connection - N - Y - 0 - 80 - 240 - - - DropTable - - SQL - ${SQL_DROP} - T - F - - T - Library_ExportCsvToTable_Connection - N - Y - 0 - 240 - 304 - - - CreateTable - - SQL - ${SQL_CREATE} - T - F - - T - Library_ExportCsvToTable_Connection - N - Y - 0 - 384 - 240 - - - Success - - SUCCESS - N - Y - 0 - 848 - 240 - - - - - ExportCsvToTable - DeleteMetadataFile - 0 - 0 - Y - Y - N - - - START - DefineMetadataVariables - 0 - 0 - Y - Y - Y - - - DefineMetadataVariables - ExtractCsvMetadataToCsv - 0 - 0 - Y - Y - N - - - ExtractCsvMetadataToCsv - DefineSqlVariables - 0 - 0 - Y - Y - N - - - DefineSqlVariables - TableExists - 0 - 0 - Y - Y - N - - - TableExists - DropTable - 0 - 0 - Y - Y - N - - - TableExists - CreateTable - 0 - 0 - Y - N - N - - - DropTable - CreateTable - 0 - 0 - Y - Y - N - - - CreateTable - ExportCsvToTable - 0 - 0 - Y - Y - N - - - DeleteMetadataFile - Success - 0 - 0 - Y - Y - N - - - - - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr deleted file mode 100644 index ebdd0ca..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExportCsvToTable/ExportCsvToTable.ktr +++ /dev/null @@ -1,353 +0,0 @@ - - - ExportCsvToTable - - - - Normal - 0 - /Library/ExportCsvToTable - - - INPUT_DELIMITER - - - - - INPUT_ENCLOSURE - - - - - INPUT_PATH - - - - - METADATA_DELIMITER - - - - - METADATA_ENCLOSURE - - - - - METADATA_PATH - - - - - OUTPUT_DRIVER - - - - - OUTPUT_PASSWORD - - - - - OUTPUT_SCHEMA - - - - - OUTPUT_TABLE - - - - - OUTPUT_URL - - - - - OUTPUT_USERNAME - - - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT - - -
- - -ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2016/10/23 18:35:54.307 - - - 2016/10/25 12:29:25.654 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - ReadMetadataFileExportCsvToTableWorkerY - - - ExportCsvToTableWorker - MetaInject - - Y - - 1 - - none - - - rep_name - - ExportCsvToTableWorker - - /Library/ExportCsvToTable - ReadCsv - - N - - - ReadCsv - FIELD_LENGTH - Y - ReadMetadataFile - Length - ReadCsv - FIELD_TRIM_TYPE - Y - ReadMetadataFile - TrimType - ReadCsv - FIELD_DECIMAL - Y - ReadMetadataFile - Decimal - ReadCsv - FIELD_NAME - Y - ReadMetadataFile - Name - ReadCsv - FIELD_FORMAT - Y - ReadMetadataFile - Format - ReadCsv - FIELD_PRECISION - Y - ReadMetadataFile - Precision - ReadCsv - FIELD_CURRENCY - Y - ReadMetadataFile - Currency - ReadCsv - FIELD_TYPE - Y - ReadMetadataFile - Type - ReadCsv - FIELD_GROUP - Y - ReadMetadataFile - Group - - - 320 - 96 - Y - - - - - ReadMetadataFile - CsvInput - - Y - - 1 - - none - - - ${METADATA_PATH} - - - N - ${METADATA_DELIMITER} - ${METADATA_ENCLOSURE} -
Y
- 50000 - Y - N - N - N - - - - Name - String - - - - - -1 - -1 - none - - - Type - String - - - - - -1 - -1 - none - - - Format - String - - - - - -1 - -1 - none - - - Length - String - - - - - -1 - -1 - none - - - Precision - String - - - - - -1 - -1 - none - - - Currency - String - - - - - -1 - -1 - none - - - Decimal - String - - - - - -1 - -1 - none - - - Group - String - - - - - -1 - -1 - none - - - TrimType - String - - - - - -1 - -1 - none - - - - - 144 - 96 - Y - - - - - - - - N - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb deleted file mode 100644 index cb33798..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.kjb +++ /dev/null @@ -1,172 +0,0 @@ - - ExtractCsvMetadataToCsv - Extracts metadata from a CSV file and saves it to another CSV file. - Extracts metadata from a CSV file and saves it to another CSV file. This job assumes that the input CSV file has at least two rows: - The first is the header row (from which the names of the fields are taken). - The second is a data row (from which the types of the fields are inferred). - - 0 - /Library/ExtractCsvMetadataToCsv - - - 2016/10/22 16:22:05.789 - - - 2016/11/02 23:00:11.519 - - - INPUT_DELIMITER - , - The delimiter character in the input CSV file. - - - INPUT_ENCLOSURE - " - The enclosure character in the input CSV file. - - - INPUT_PATH - - The path to the input CSV file. - - - OUTPUT_DELIMITER - , - The delimiter character in the output metadata (CSV) file. - - - OUTPUT_ENCLOSURE - " - The delimiter character in the enclosure metadata (CSV) file. - - - OUTPUT_PATH - - The path of the new metadata (CSV) file. - - - - - - -
- - - -ID_JOBYID_JOBCHANNEL_IDYCHANNEL_IDJOBNAMEYJOBNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERSTART_JOB_ENTRYNSTART_JOB_ENTRYCLIENTNCLIENT - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEJOBNAMEYTRANSNAMEJOBENTRYNAMEYSTEPNAMELINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSRESULTYRESULTNR_RESULT_ROWSYNR_RESULT_ROWSNR_RESULT_FILESYNR_RESULT_FILESLOG_FIELDNLOG_FIELDCOPY_NRNCOPY_NR - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - N - - - - START - - SPECIAL - Y - N - N - 0 - 0 - 60 - 12 - 0 - 1 - 1 - N - Y - 0 - 80 - 48 - - - ExtractCsvMetadataToCsv - - TRANS - rep_name - - - ExtractCsvMetadataToCsv - /Library/ExtractCsvMetadataToCsv - N - N - N - N - N - N - - - N - N - Basic - N - - N - Y - N - N - N - N - INPUT_DELIMITER - - ${INPUT_DELIMITER} - INPUT_ENCLOSURE - - ${INPUT_ENCLOSURE} - INPUT_PATH - - ${INPUT_PATH} - OUTPUT_DELIMITER - - ${OUTPUT_DELIMITER} - OUTPUT_ENCLOSURE - - ${OUTPUT_ENCLOSURE} - OUTPUT_PATH - - ${OUTPUT_PATH} - N - Y - 0 - 256 - 48 - - - Success - - SUCCESS - N - Y - 0 - 432 - 48 - - - - - START - ExtractCsvMetadataToCsv - 0 - 0 - Y - Y - Y - - - ExtractCsvMetadataToCsv - Success - 0 - 0 - Y - Y - N - - - - - - diff --git a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr b/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr deleted file mode 100644 index 8ca208c..0000000 --- a/pentaho/kissflow/BlocPowerETL/Library/ExtractCsvMetadataToCsv/ExtractCsvMetadataToCsv.ktr +++ /dev/null @@ -1,550 +0,0 @@ - - - ExtractCsvMetadataToCsv - - - - Normal - 0 - /Library/ExtractCsvMetadataToCsv - - - INPUT_DELIMITER - , - - - - INPUT_ENCLOSURE - " - - - - INPUT_PATH - - - - - OUTPUT_DELIMITER - , - - - - OUTPUT_ENCLOSURE - " - - - - OUTPUT_PATH - - - - - - - -
- - - -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDTRANSNAMEYTRANSNAMESTATUSYSTATUSLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSSTARTDATEYSTARTDATEENDDATEYENDDATELOGDATEYLOGDATEDEPDATEYDEPDATEREPLAYDATEYREPLAYDATELOG_FIELDYLOG_FIELDEXECUTING_SERVERNEXECUTING_SERVEREXECUTING_USERNEXECUTING_USERCLIENTNCLIENT - - -
- - -ID_BATCHYID_BATCHSEQ_NRYSEQ_NRLOGDATEYLOGDATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSINPUT_BUFFER_ROWSYINPUT_BUFFER_ROWSOUTPUT_BUFFER_ROWSYOUTPUT_BUFFER_ROWS - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATELOGGING_OBJECT_TYPEYLOGGING_OBJECT_TYPEOBJECT_NAMEYOBJECT_NAMEOBJECT_COPYYOBJECT_COPYREPOSITORY_DIRECTORYYREPOSITORY_DIRECTORYFILENAMEYFILENAMEOBJECT_IDYOBJECT_IDOBJECT_REVISIONYOBJECT_REVISIONPARENT_CHANNEL_IDYPARENT_CHANNEL_IDROOT_CHANNEL_IDYROOT_CHANNEL_ID - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATETRANSNAMEYTRANSNAMESTEPNAMEYSTEPNAMESTEP_COPYYSTEP_COPYLINES_READYLINES_READLINES_WRITTENYLINES_WRITTENLINES_UPDATEDYLINES_UPDATEDLINES_INPUTYLINES_INPUTLINES_OUTPUTYLINES_OUTPUTLINES_REJECTEDYLINES_REJECTEDERRORSYERRORSLOG_FIELDNLOG_FIELD - - -
- -ID_BATCHYID_BATCHCHANNEL_IDYCHANNEL_IDLOG_DATEYLOG_DATEMETRICS_DATEYMETRICS_DATEMETRICS_CODEYMETRICS_CODEMETRICS_DESCRIPTIONYMETRICS_DESCRIPTIONMETRICS_SUBJECTYMETRICS_SUBJECTMETRICS_TYPEYMETRICS_TYPEMETRICS_VALUEYMETRICS_VALUE - - - -
- - 0.0 - 0.0 - - 10000 - 50 - 50 - N - Y - 50000 - Y - - N - 1000 - 100 - - - - - - - - - - - 2016/10/22 13:07:22.373 - - - 2016/11/03 14:01:30.830 - H4sIAAAAAAAAAAMAAAAAAAAAAAA= - N - - - - - KeepHeaderOnlyBuildRowForEachFieldY - BuildRowForEachFieldCleanRowSetY - CleanRowSetPickFieldNameY - PickFieldNameAddFieldTypeAsStringY - AddFieldTypeAsStringExtractCsvMetadataToCsvWorkerY - ExtractCsvMetadataToCsvWorkerWriteCsvY - ReadLinesFromCsvKeepHeaderOnlyY - - - AddFieldTypeAsString - Constant - - Y - - 1 - - none - - - - - FieldType - String - - - - - String - -1 - -1 - N - - - - - 240 - 224 - Y - - - - - BuildRowForEachField - ScriptValueMod - - Y - - 1 - - none - - - N - 9 - 1 - ScriptStart - // DEFAULT VALUES: var FieldName = null; // AUXILIARY FUNCTIONS: function SplitLine(line) { var delimiter = getVariable("INPUT_DELIMITER",","); var enclosure = getVariable("INPUT_ENCLOSURE", "\""); var fields = []; var currentField = ""; var enclosureSeen = false; for (var i = 0; i < line.length; i++) { var character = line.charAt(i); if (enclosureSeen) { if (character == enclosure) { if (i < line.length - 1 && line.charAt(i + 1) == enclosure) { currentField += enclosure; i += 1; } else { enclosureSeen = false; } } else { currentField += character; } } else { if (character == delimiter) { fields.push(currentField); currentField = ""; } else if (character == enclosure) { enclosureSeen = true; } else { currentField += character; } } } fields.push(currentField); currentField = ""; enclosureSeen = false; var regexp = new RegExp(enclosure, "g"); for (var i = 0; i < fields.length; i++) { fields[i] = fields[i].trim(); if (enclosure == "\"") { fields[i] = fields[i].replace(regexp, "'"); } } return fields; } - 0 - ScriptLoop - // LOOP CODE: var fields = SplitLine(LineText); for (var i = 0; i < fields.length; i++) { var newRow = createRowCopy(getOutputRowMeta().size()); var rowIndex = getInputRowMeta().size(); // FieldName newRow[rowIndex] = fields[i]; putRow(newRow); } - FieldName - FieldName - String - -1 - -1 - N - - - 416 - 64 - Y - - - - - CleanRowSet - FilterRows - - Y - - 1 - - none - - -PickFieldName - - - - N - FieldName - IS NOT NULL - - - - - - 624 - 64 - Y - - - - - ExtractCsvMetadataToCsvWorker - MetaInject - - Y - - 1 - - none - - - rep_name - - ExtractCsvMetadataToCsvWorker - - /Library/ExtractCsvMetadataToCsv - PickMetadataFields - Name - String - -1 - -1 - Type - String - -1 - -1 - Format - String - -1 - -1 - Length - String - -1 - -1 - Precision - String - -1 - -1 - Currency - String - -1 - -1 - Decimal - String - -1 - -1 - Group - String - -1 - -1 - TrimType - String - -1 - -1 - - N - - - ReadCsv - FIELD_NAME - Y - AddFieldTypeAsString - FieldName - ReadCsv - FIELD_TYPE - Y - AddFieldTypeAsString - FieldType - - - 448 - 224 - Y - - - - - KeepHeaderOnly - FilterRows - - Y - - 1 - - none - - -BuildRowForEachField - - - - N - LineNumber - = - - constantInteger1-10N#;-# - - - - 240 - 64 - Y - - - - - PickFieldName - SelectValues - - Y - - 1 - - none - - - FieldName - - -2 - -2 - N - - - 80 - 224 - Y - - - - - ReadLinesFromCsv - CsvInput - - Y - - 1 - - none - - - ${INPUT_PATH} - - LineNumber - N - $[00] - $[01] -
N
- 50000 - Y - N - N - N - - - - LineText - String - - - - - -1 - -1 - none - - - - - 80 - 64 - Y - - - - - WriteCsv - TextFileOutput - - Y - - 1 - - none - - - ${OUTPUT_DELIMITER} - ${OUTPUT_ENCLOSURE} - N - N -
Y
-
N
- DOS - None - windows-1252 - - N - - Y - - ${OUTPUT_PATH} - N - N - N - - N - N - N - N - N - N - - Y - N - N - 0 - - - - Name - String - - - - - - none - -1 - -1 - - - Type - String - - - - - - none - -1 - -1 - - - Format - String - - - - - - none - -1 - -1 - - - Length - String - - - - - - none - -1 - -1 - - - Precision - String - - - - - - none - -1 - -1 - - - Currency - String - - - - - - none - -1 - -1 - - - Decimal - String - - - - - - none - -1 - -1 - - - Group - String - - - - - - none - -1 - -1 - - - TrimType - String - - - - - - none - -1 - -1 - - - - - 608 - 224 - Y - - - - - - - - N - - diff --git a/pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb b/pentaho/kissflow/BlocPowerETL/Tester.kjb similarity index 97% rename from pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb rename to pentaho/kissflow/BlocPowerETL/Tester.kjb index c5ab7ed..63c73bf 100644 --- a/pentaho/kissflow/BlocPowerETL/Programs/Tester/Tester.kjb +++ b/pentaho/kissflow/BlocPowerETL/Tester.kjb @@ -1,3 +1,4 @@ + Tester Tester @@ -7,7 +8,7 @@ Dependencies: - /Library (folder) 0 - /Programs/Tester + / - 2016/10/25 15:13:28.377 - @@ -313,11 +314,10 @@ Dependencies: ExportCsvInDirectoryToDatabase JOB - rep_name + filename - - ExportCsvInDirectoryToDatabase - /Library/ExportCsvInDirectoryToDatabase + ${Internal.Entry.Current.Directory}/ExportCsvInDirectoryToDatabase.kjb + N N N diff --git a/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr index 58250f7..6777817 100644 --- a/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr +++ b/pentaho/kissflow/BlocPowerETL/kissflow_script.ktr @@ -1,3 +1,4 @@ + kissflow_script @@ -452,39 +453,6 @@ Y - - Execute a process - ExecProcess - - Y - - 1 - - none - - - command - Result output - Error output - Exit value - Y - - N - - - - - - - - - - - 800 - 96 - Y - - Create command Calculator @@ -540,8 +508,8 @@ - Split apps string into rows - SplitFieldToRows3 + Execute a process + ExecProcess Y @@ -550,13 +518,15 @@ none - apps_string - , - app_name - N - - Y - N + command + Result output + Error output + Exit value + Y + + N + + @@ -565,7 +535,7 @@ - 448 + 800 96 Y @@ -620,6 +590,37 @@ Y + + Split apps string into rows + SplitFieldToRows3 + + Y + + 1 + + none + + + apps_string + , + app_name + N + + Y + N + + + + + + + + + 448 + 96 + Y + + diff --git a/pentaho/kissflow/BlocPowerETL/master_job.kjb b/pentaho/kissflow/BlocPowerETL/master_job.kjb index fce7ae3..4b2c5f5 100644 --- a/pentaho/kissflow/BlocPowerETL/master_job.kjb +++ b/pentaho/kissflow/BlocPowerETL/master_job.kjb @@ -1,3 +1,4 @@ + master_job @@ -8,7 +9,7 @@ - 2017/09/13 16:16:41.263 - - 2017/09/13 16:48:42.729 + 2017/09/15 18:07:52.592 @@ -347,17 +348,16 @@ CSV to DB JOB - rep_name + filename - - Tester - ${Internal.Entry.Current.Directory}Programs${Internal.Entry.Current.Directory}Tester + ${Internal.Entry.Current.Directory}/Tester.kjb + N N N - Y - /home/conrad/Projects/Blocpower/oltp/Kissflow/logs/kettlelog - + N + + log N N Debug diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py old mode 100644 new mode 100755 index b596d47..47a6a3f --- a/pentaho/kissflow/kissflow.py +++ b/pentaho/kissflow/kissflow.py @@ -6,6 +6,7 @@ import sys import time import os + LOGGING_PATH = os.environ['KISSFLOW_LOGGING_PATH'] CSV_PATH = os.environ['KISSFLOW_CSV_PATH'] KISSFLOW_API_KEY = os.environ['KISSFLOW_API_KEY'] -- GitLab From 47a7fb1fcd6a2be4010fb5bbc5d4185b08d38e62 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 Sep 2017 15:54:24 -0400 Subject: [PATCH 04/11] Turn off testing --- pentaho/kissflow/kissflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py index 47a6a3f..b39896c 100755 --- a/pentaho/kissflow/kissflow.py +++ b/pentaho/kissflow/kissflow.py @@ -13,7 +13,7 @@ KISSFLOW_API_KEY = os.environ['KISSFLOW_API_KEY'] KISSFLOW_API_URL = os.environ['KISSFLOW_API_URL'] APP_NAME = sys.argv[1] # If testing is true, only grab first 50 entries -TESTING = True +TESTING = False PAGE_SIZE = 50 start = time.time() -- GitLab From e6a0be5839710a2808a12af68463dba0189d9e40 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 Sep 2017 16:01:05 -0400 Subject: [PATCH 05/11] Update readme --- Dockerfile | 1 - README.md | 27 ++++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e9ea9f3..0eff331 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,6 @@ RUN mkdir -p $KETTLE_HOME/.kettle RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ -COPY config/repositories.xml $KETTLE_HOME/.kettle/repositories.xml COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml COPY pentaho/ $KETTLE_HOME/jobs/ COPY config/carte_master_config.xml /pentaho-di/carte_config.xml diff --git a/README.md b/README.md index e01e50e..0be7744 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,27 @@ CARTE_MASTER_PASSWORD Variables for `kettle.properties` must be prefixed with `KP_`. -curl -X POST 'localhost:9081/kettle/executeJob/?job=/pentaho-di/jobs/kissflow/BlocPowerETL/master_job.kjb&level=Debug' --user conrad:password -docker run -p 9081:9081 --env-file .env pentaho -docker build -t pentaho . +BUILD DOCKER: docker build -t pentaho . +RUN DOCKER: docker run -p 9081:9081 --env-file .env pentaho +CREATE A REQUEST: curl -X POST 'localhost:9081/kettle/executeJob/?job=/path/to/job/JOB_NAME.kjb&level=Debug' --user user:password + +## Kissflow + +Variables required for kissflow, with examples for nonsensitive data: +``` +KP_OUTPUT_DRIVER=org.postgresql.Driver +KP_OUTPUT_SCHEMA=public +KP_OUTPUT_PASSWORD +KP_OUTPUT_USERNAME +KP_OUTPUT_URL=jdbc\:postgresql\://HOSTNAME\:PORT/DATABASE_NAME +KP_KISSFLOW_SCRIPT_PATH=/pentaho-di/jobs/kissflow/run_kissflow.sh +KP_KISSFLOW_APPS=Lighting,CBRA,Sensors +KP_INPUT_DELIMITER=, +KP_INPUT_DIRECTORY=/pentaho-di/jobs/kissflow/kissflow_apps +KP_INPUT_ENCLOSURE=\" + +KISSFLOW_API_KEY +KISSFLOW_API_URL=https://kf-0003173.appspot.com/api/1/{}/list/p{}/{} +KISSFLOW_LOGGING_PATH=/pentaho-di/jobs/kissflow/logs/pylog-{} +KISSFLOW_CSV_PATH=/pentaho-di/jobs/kissflow/kissflow_apps/app_{}.csv +``` -- GitLab From 580648d4d6f316b1d04295296964c9442e9fddbc Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 Sep 2017 16:04:54 -0400 Subject: [PATCH 06/11] Remove repository log --- pentaho/kissflow/BlocPowerETL/repository.log | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 pentaho/kissflow/BlocPowerETL/repository.log diff --git a/pentaho/kissflow/BlocPowerETL/repository.log b/pentaho/kissflow/BlocPowerETL/repository.log deleted file mode 100644 index 6672bd1..0000000 --- a/pentaho/kissflow/BlocPowerETL/repository.log +++ /dev/null @@ -1,10 +0,0 @@ -Save database : Modification by user -Save repository element : Library_ExportCsvToTable_Connection : Modification by user -Save database : Modification by user -Save repository element : Library_ExportCsvToTable_Connection : Modification by user -Save database : Modification by user -Save repository element : Library_ExportCsvToTable_Connection : Modification by user -Save database : Modification by user -Save repository element : Library_ExportCsvToTable_Connection : Modification by user -Save database : Modification by user -Save repository element : Library_ExportCsvToTable_Connection : Modification by user -- GitLab From 895f06327219a9aebace9a09993fe9e0e5d301d4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 19 Sep 2017 12:27:56 -0400 Subject: [PATCH 07/11] Catch error related to ascii issues --- pentaho/kissflow/kissflow.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pentaho/kissflow/kissflow.py b/pentaho/kissflow/kissflow.py index b39896c..a53ac2f 100755 --- a/pentaho/kissflow/kissflow.py +++ b/pentaho/kissflow/kissflow.py @@ -17,7 +17,7 @@ TESTING = False PAGE_SIZE = 50 start = time.time() -log_file = open(LOGGING_PATH.format(APP_NAME), 'a') +log_file = open(LOGGING_PATH.format(APP_NAME), 'w') def log(s): log_file.write('{}: {}\n'.format( str(datetime.now()), @@ -75,7 +75,10 @@ with open(CSV_PATH.format(APP_NAME), 'wb') as csvfile: if type(field) is list: field = json.dumps(field) elif type(field) is not str: - field = str(field) + try: + field = str(field) + except Exception as e: + field = str(field.encode('ascii', 'ignore')) row.append(field.encode('ascii', 'ignore')) else: -- GitLab From 8bd3095a3c670859d632019369eebd2e0403e1e1 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 19 Sep 2017 12:44:32 -0400 Subject: [PATCH 08/11] Create log and csv folders in docker file --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0eff331..7e8bbd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,8 @@ RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml COPY pentaho/ $KETTLE_HOME/jobs/ +RUN mkdir $KETTLE_HOME/jobs/kissflow/logs +RUN mkdir $KETTLE_HOME/jobs/kissflow/kissflow_apps COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / RUN chmod +x /entrypoint.sh -- GitLab From 813abdf2523d4418dba79425ca908e455be03e39 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 19 Sep 2017 15:44:45 -0400 Subject: [PATCH 09/11] Remove kissflow repository --- config/carte_master_config.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/carte_master_config.xml b/config/carte_master_config.xml index 52545ca..97798e2 100644 --- a/config/carte_master_config.xml +++ b/config/carte_master_config.xml @@ -7,9 +7,6 @@ CARTE_MASTER_USER CARTE_MASTER_PASSWORD - - kissflow - 10000 1440 -- GitLab From a5d8eab82c90a3165b62d72e70dfd6934db9bdea Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 21 Sep 2017 10:59:55 -0400 Subject: [PATCH 10/11] Use /var/log for logs --- Dockerfile | 7 +++---- README.md | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e8bbd7..8152c10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,7 @@ ENV PDI_RELEASE=7.1 \ RUN apt-get update && apt-get install -y \ libwebkitgtk-1.0.0 \ python \ - python-pip \ - vim + python-pip RUN pip install requests @@ -26,8 +25,8 @@ RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml COPY pentaho/ $KETTLE_HOME/jobs/ -RUN mkdir $KETTLE_HOME/jobs/kissflow/logs -RUN mkdir $KETTLE_HOME/jobs/kissflow/kissflow_apps +RUN mkdir /var/log/kissflow +RUN mkdir -p $KETTLE_HOME/jobs/kissflow/kissflow_apps COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / RUN chmod +x /entrypoint.sh diff --git a/README.md b/README.md index 0be7744..9c0bb8a 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,6 @@ KP_INPUT_ENCLOSURE=\" KISSFLOW_API_KEY KISSFLOW_API_URL=https://kf-0003173.appspot.com/api/1/{}/list/p{}/{} -KISSFLOW_LOGGING_PATH=/pentaho-di/jobs/kissflow/logs/pylog-{} +KISSFLOW_LOGGING_PATH=/var/log/kissflow/pylog-{} KISSFLOW_CSV_PATH=/pentaho-di/jobs/kissflow/kissflow_apps/app_{}.csv ``` -- GitLab From 676ebfcf35e971fc4da6123fb0bfa0033aa659b6 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 21 Sep 2017 11:01:08 -0400 Subject: [PATCH 11/11] Move shared.xml copy command to other copy commands --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8152c10..9ef3c8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,12 +23,12 @@ RUN mkdir -p $KETTLE_HOME/.kettle RUN mkdir -p $KETTLE_HOME/jobs COPY pentaho/eplus/* $KETTLE_HOME/.kettle/ -COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml COPY pentaho/ $KETTLE_HOME/jobs/ RUN mkdir /var/log/kissflow RUN mkdir -p $KETTLE_HOME/jobs/kissflow/kissflow_apps COPY config/carte_master_config.xml /pentaho-di/carte_config.xml COPY config/entrypoint.sh / +COPY config/shared.xml $KETTLE_HOME/.kettle/shared.xml RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] -- GitLab