Entrega borrador
This commit is contained in:
juanlopez707
571
ETL/Get_CSV_files.ktr
Normal file
571
ETL/Get_CSV_files.ktr
Normal file
@ -0,0 +1,571 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<transformation>
|
||||
<info>
|
||||
<name>Get_CSV_files</name>
|
||||
<description>Obtiene el listado de ficheros CSV con los datos a procesar</description>
|
||||
<extended_description>Obtiene el listado de ficheros CSV con los datos a procesar</extended_description>
|
||||
<trans_version/>
|
||||
<trans_type>Normal</trans_type>
|
||||
<trans_status>0</trans_status>
|
||||
<directory>/</directory>
|
||||
<parameters>
|
||||
</parameters>
|
||||
<log>
|
||||
<trans-log-table>
|
||||
<connection/>
|
||||
<schema/>
|
||||
<table/>
|
||||
<size_limit_lines/>
|
||||
<interval/>
|
||||
<timeout_days/>
|
||||
<field>
|
||||
<id>ID_BATCH</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ID_BATCH</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>CHANNEL_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>TRANSNAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>TRANSNAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STATUS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STATUS</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_READ</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_READ</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_WRITTEN</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_WRITTEN</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_UPDATED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_UPDATED</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_INPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_INPUT</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_OUTPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_OUTPUT</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_REJECTED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_REJECTED</name>
|
||||
<subject/>
|
||||
</field>
|
||||
<field>
|
||||
<id>ERRORS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ERRORS</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STARTDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STARTDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>ENDDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ENDDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOGDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOGDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>DEPDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>DEPDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>REPLAYDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>REPLAYDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOG_FIELD</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOG_FIELD</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>EXECUTING_SERVER</id>
|
||||
<enabled>N</enabled>
|
||||
<name>EXECUTING_SERVER</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>EXECUTING_USER</id>
|
||||
<enabled>N</enabled>
|
||||
<name>EXECUTING_USER</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>CLIENT</id>
|
||||
<enabled>N</enabled>
|
||||
<name>CLIENT</name>
|
||||
</field>
|
||||
</trans-log-table>
|
||||
<perf-log-table>
|
||||
<connection/>
|
||||
<schema/>
|
||||
<table/>
|
||||
<interval/>
|
||||
<timeout_days/>
|
||||
<field>
|
||||
<id>ID_BATCH</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ID_BATCH</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>SEQ_NR</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>SEQ_NR</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOGDATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOGDATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>TRANSNAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>TRANSNAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STEPNAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STEPNAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STEP_COPY</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STEP_COPY</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_READ</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_READ</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_WRITTEN</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_WRITTEN</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_UPDATED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_UPDATED</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_INPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_INPUT</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_OUTPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_OUTPUT</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_REJECTED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_REJECTED</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>ERRORS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ERRORS</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>INPUT_BUFFER_ROWS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>INPUT_BUFFER_ROWS</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>OUTPUT_BUFFER_ROWS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>OUTPUT_BUFFER_ROWS</name>
|
||||
</field>
|
||||
</perf-log-table>
|
||||
<channel-log-table>
|
||||
<connection/>
|
||||
<schema/>
|
||||
<table/>
|
||||
<timeout_days/>
|
||||
<field>
|
||||
<id>ID_BATCH</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ID_BATCH</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>CHANNEL_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOG_DATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOG_DATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOGGING_OBJECT_TYPE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOGGING_OBJECT_TYPE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>OBJECT_NAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>OBJECT_NAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>OBJECT_COPY</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>OBJECT_COPY</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>REPOSITORY_DIRECTORY</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>REPOSITORY_DIRECTORY</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>FILENAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>FILENAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>OBJECT_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>OBJECT_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>OBJECT_REVISION</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>OBJECT_REVISION</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>PARENT_CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>PARENT_CHANNEL_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>ROOT_CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ROOT_CHANNEL_ID</name>
|
||||
</field>
|
||||
</channel-log-table>
|
||||
<step-log-table>
|
||||
<connection/>
|
||||
<schema/>
|
||||
<table/>
|
||||
<timeout_days/>
|
||||
<field>
|
||||
<id>ID_BATCH</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ID_BATCH</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>CHANNEL_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOG_DATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOG_DATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>TRANSNAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>TRANSNAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STEPNAME</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STEPNAME</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>STEP_COPY</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>STEP_COPY</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_READ</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_READ</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_WRITTEN</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_WRITTEN</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_UPDATED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_UPDATED</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_INPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_INPUT</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_OUTPUT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_OUTPUT</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LINES_REJECTED</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LINES_REJECTED</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>ERRORS</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ERRORS</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOG_FIELD</id>
|
||||
<enabled>N</enabled>
|
||||
<name>LOG_FIELD</name>
|
||||
</field>
|
||||
</step-log-table>
|
||||
<metrics-log-table>
|
||||
<connection/>
|
||||
<schema/>
|
||||
<table/>
|
||||
<timeout_days/>
|
||||
<field>
|
||||
<id>ID_BATCH</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>ID_BATCH</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>CHANNEL_ID</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>CHANNEL_ID</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>LOG_DATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>LOG_DATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_DATE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_DATE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_CODE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_CODE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_DESCRIPTION</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_DESCRIPTION</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_SUBJECT</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_SUBJECT</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_TYPE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_TYPE</name>
|
||||
</field>
|
||||
<field>
|
||||
<id>METRICS_VALUE</id>
|
||||
<enabled>Y</enabled>
|
||||
<name>METRICS_VALUE</name>
|
||||
</field>
|
||||
</metrics-log-table>
|
||||
</log>
|
||||
<maxdate>
|
||||
<connection/>
|
||||
<table/>
|
||||
<field/>
|
||||
<offset>0.0</offset>
|
||||
<maxdiff>0.0</maxdiff>
|
||||
</maxdate>
|
||||
<size_rowset>10000</size_rowset>
|
||||
<sleep_time_empty>50</sleep_time_empty>
|
||||
<sleep_time_full>50</sleep_time_full>
|
||||
<unique_connections>N</unique_connections>
|
||||
<feedback_shown>Y</feedback_shown>
|
||||
<feedback_size>50000</feedback_size>
|
||||
<using_thread_priorities>Y</using_thread_priorities>
|
||||
<shared_objects_file/>
|
||||
<capture_step_performance>N</capture_step_performance>
|
||||
<step_performance_capturing_delay>1000</step_performance_capturing_delay>
|
||||
<step_performance_capturing_size_limit>100</step_performance_capturing_size_limit>
|
||||
<dependencies>
|
||||
</dependencies>
|
||||
<partitionschemas>
|
||||
</partitionschemas>
|
||||
<slaveservers>
|
||||
</slaveservers>
|
||||
<clusterschemas>
|
||||
</clusterschemas>
|
||||
<created_user>-</created_user>
|
||||
<created_date>2023/06/09 19:18:38.140</created_date>
|
||||
<modified_user>-</modified_user>
|
||||
<modified_date>2023/06/09 19:18:38.140</modified_date>
|
||||
<key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
|
||||
<is_key_private>N</is_key_private>
|
||||
</info>
|
||||
<notepads>
|
||||
</notepads>
|
||||
<order>
|
||||
<hop>
|
||||
<from>Get file names</from>
|
||||
<to>Sort rows</to>
|
||||
<enabled>Y</enabled>
|
||||
</hop>
|
||||
<hop>
|
||||
<from>Sort rows</from>
|
||||
<to>Set files in result</to>
|
||||
<enabled>Y</enabled>
|
||||
</hop>
|
||||
</order>
|
||||
<step>
|
||||
<name>Get file names</name>
|
||||
<type>GetFileNames</type>
|
||||
<description/>
|
||||
<distribute>Y</distribute>
|
||||
<custom_distribution/>
|
||||
<copies>1</copies>
|
||||
<partitioning>
|
||||
<method>none</method>
|
||||
<schema_name/>
|
||||
</partitioning>
|
||||
<filter>
|
||||
<filterfiletype>all_files</filterfiletype>
|
||||
</filter>
|
||||
<doNotFailIfNoFile>N</doNotFailIfNoFile>
|
||||
<rownum>N</rownum>
|
||||
<isaddresult>Y</isaddresult>
|
||||
<filefield>N</filefield>
|
||||
<rownum_field/>
|
||||
<filename_Field/>
|
||||
<wildcard_Field/>
|
||||
<exclude_wildcard_Field/>
|
||||
<dynamic_include_subfolders>N</dynamic_include_subfolders>
|
||||
<limit>0</limit>
|
||||
<file>
|
||||
<name>${DATA_DIR}</name>
|
||||
<filemask>data.*\.csv</filemask>
|
||||
<exclude_filemask/>
|
||||
<file_required>N</file_required>
|
||||
<include_subfolders>N</include_subfolders>
|
||||
</file>
|
||||
<attributes/>
|
||||
<cluster_schema/>
|
||||
<remotesteps>
|
||||
<input>
|
||||
</input>
|
||||
<output>
|
||||
</output>
|
||||
</remotesteps>
|
||||
<GUI>
|
||||
<xloc>80</xloc>
|
||||
<yloc>192</yloc>
|
||||
<draw>Y</draw>
|
||||
</GUI>
|
||||
</step>
|
||||
<step>
|
||||
<name>Set files in result</name>
|
||||
<type>FilesToResult</type>
|
||||
<description/>
|
||||
<distribute>Y</distribute>
|
||||
<custom_distribution/>
|
||||
<copies>1</copies>
|
||||
<partitioning>
|
||||
<method>none</method>
|
||||
<schema_name/>
|
||||
</partitioning>
|
||||
<filename_field>filename</filename_field>
|
||||
<file_type>GENERAL</file_type>
|
||||
<attributes/>
|
||||
<cluster_schema/>
|
||||
<remotesteps>
|
||||
<input>
|
||||
</input>
|
||||
<output>
|
||||
</output>
|
||||
</remotesteps>
|
||||
<GUI>
|
||||
<xloc>288</xloc>
|
||||
<yloc>192</yloc>
|
||||
<draw>Y</draw>
|
||||
</GUI>
|
||||
</step>
|
||||
<step>
|
||||
<name>Sort rows</name>
|
||||
<type>SortRows</type>
|
||||
<description/>
|
||||
<distribute>Y</distribute>
|
||||
<custom_distribution/>
|
||||
<copies>1</copies>
|
||||
<partitioning>
|
||||
<method>none</method>
|
||||
<schema_name/>
|
||||
</partitioning>
|
||||
<directory>%%java.io.tmpdir%%</directory>
|
||||
<prefix>out</prefix>
|
||||
<sort_size>1000000</sort_size>
|
||||
<free_memory/>
|
||||
<compress>N</compress>
|
||||
<compress_variable/>
|
||||
<unique_rows>N</unique_rows>
|
||||
<fields>
|
||||
<field>
|
||||
<name>filename</name>
|
||||
<ascending>Y</ascending>
|
||||
<case_sensitive>N</case_sensitive>
|
||||
<collator_enabled>N</collator_enabled>
|
||||
<collator_strength>0</collator_strength>
|
||||
<presorted>N</presorted>
|
||||
</field>
|
||||
</fields>
|
||||
<attributes/>
|
||||
<cluster_schema/>
|
||||
<remotesteps>
|
||||
<input>
|
||||
</input>
|
||||
<output>
|
||||
</output>
|
||||
</remotesteps>
|
||||
<GUI>
|
||||
<xloc>192</xloc>
|
||||
<yloc>192</yloc>
|
||||
<draw>Y</draw>
|
||||
</GUI>
|
||||
</step>
|
||||
<step_error_handling>
|
||||
</step_error_handling>
|
||||
<slave-step-copy-partition-distribution>
|
||||
</slave-step-copy-partition-distribution>
|
||||
<slave_transformation>N</slave_transformation>
|
||||
<attributes/>
|
||||
</transformation>
|
||||
Reference in New Issue
Block a user