generator.pm

普通的ETL工具

			$self->user_prog()->comment(" ---> Configuration >> statistics <<");
			$self->user_prog()->comment("");
			$self->user_prog()->code("use Benchmark;");
			$self->user_prog()->code("my \$benchmark_start = new Benchmark;");
		}

		my $r=0;
		map
		(
			$self->user_prog()->code("my \$_IRVAL_@{[ ++$r ]}; # Input repeater $r value"),
			$self->main()->repeat_record_on_input()->fields()->to_array()
		)
		if ($self->main()->user_sections()->repeat_record_on_input()->size());

		$r=0;
		map
		(
			$self->user_prog()->code("my \$_ORVAL_@{[ ++$r ]}; # Output repeater $r value"),
			$self->main()->repeat_record_on_output()->fields()->to_array()
		)
		if ($self->main()->user_sections()->repeat_record_on_output()->size());
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::OpenLocal; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'open_local');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		foreach (grep(!$_->shared(), $self->_get_datasets()))
		{
			$self->user_prog()->comment("");
			$self->user_prog()->comment(" ---> @{[ 
				ucfirst($_->datasource()->io_name()) 
				]}: @{[ $_->datasource()->fdname() ]} (@{[ 
				$_->name() ]}:@{[ $_->datasource()->datasource_name() ]})");
			$self->user_prog()->comment("");
			$_->code_open($self->user_prog()); 
		}
		# Output Header
		foreach 
		(
			grep
			(
				defined($_) && !$_->shared() && $_->print_header(),
				(
					$self->main()->output()->output_dataset(),
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_output()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_output()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_output()->to_array()),
				)
			)
		) 
		{
			$self->catalogue()->code_segments()->print_output_header()->new(
				main => $self->main(), user_prog => $self->user_prog(), dataset => $_)
				->prepare();
		}
		# Input Header
		foreach 
		(
			grep
			(
				defined($_) && !$_->shared() && $_->print_header(),
				(
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_input()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_input()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_input()->to_array()),
				)
			)
		) 
		{
			$self->catalogue()->code_segments()->print_input_header()->new(
				main => $self->main(), user_prog => $self->user_prog(), dataset => $_)
				->prepare();
		}
	}
	sub prepare_finish
	{
		my $self = shift;
		$self->program_info();
		foreach (reverse grep(!$_->shared(), $self->_get_datasets())) {
			$_->code_close($self->user_prog());
		}
	}
	sub _get_datasets
	{
		my $self = shift;
		return (
			grep
			(
				defined($_),
				(
					map($_->input_dataset(), $self->main()->user_joins()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_input()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_output()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_input()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_output()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_input()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_output()->to_array()),
					$self->main()->input()->input_dataset(),
					$self->main()->output()->output_dataset()
				)
			)
		);	
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::Main; #--> Input
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'main');
	use Class::STL::ClassMembers::Constructor; 
	sub new_extra
	{
		my $self = shift;
	
		$self->segments()->push_back($self->catalogue()->code_segments()->main_while_loop()->new(@_));

		$self->segments()->push_back($self->catalogue()->code_segments()->input_record_counter()->new(@_));

		$self->segments()->push_back($self->catalogue()->code_segments()->input_join()->new(@_))
			if ($self->main()->user_joins()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->input_repeater()->new(@_))
			if ($self->main()->user_sections()->repeat_record_on_input()->size());
	
		# TODO: this should be right after main_while_loop
#<		$self->segments()->push_back($self->catalogue()->code_segments()->input_record_counter()->new(@_));
	
		$self->segments()->push_back($self->catalogue()->code_segments()->input_record_limit()->new(@_))
			if ($self->main()->config()->input_record_limit());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->derived_input_fields()->new(@_));
	
		$self->segments()->push_back($self->catalogue()->code_segments()->field_process_input()->new(@_))
			if ($self->main()->user_sections()->field_process_on_input()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->input_filter()->new(@_))
			if ($self->main()->user_sections()->filter_on_input()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->reject_input_record()->new(@_))
			if ($self->main()->user_sections()->reject_record_on_input()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->divert_input_record()->new(@_))
			if ($self->main()->user_sections()->divert_record_on_input()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->copy_input_record()->new(@_))
			if ($self->main()->user_sections()->copy_record_on_input()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->grouping_break_control()->new(@_))
			if ($self->main()->user_sections()->group_by()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->dedup_on_input()->new(@_))
			if ($self->main()->user_sections()->group_by()->size()); #< && !$self->main()->config()->hash());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->main_body()->new(@_));
	
		$self->segments()->push_back($self->catalogue()->code_segments()->prepare_transfer_fields()->new(@_))
			if ($self->main()->output()->transfer_fields()->size());
	
		$self->segments()->push_back($self->catalogue()->code_segments()->output()->new(@_))
			if ($self->main()->user_sections()->group_by()->size() == 0);
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::MainWhileLoop; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'main_while_loop');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		$self->catalogue()->code_segments()->verbose_msg()->new(
			target_name => $self->main()->pequel_name(),
			user_prog => $self->user_prog(), 
			msg_text => "Start",
		)->prepare() if ($self->main()->config()->verbose()); 
		$self->main()->input()->input_dataset()->code_prepare($self->user_prog()); 
		$self->user_prog()->newline_off();

		$self->user_prog()->code("while (");
		$self->main()->input()->input_dataset()->code_read($self->user_prog(), 
		   grep(!defined($_->calc_exp()), $self->main()->input()->input_fields()->to_array()));
		$self->user_prog()->newline_on();
		$self->user_prog()->code(")");
		$self->user_prog()->open_block(); # main while loop
			$self->main()->input()->input_dataset()->code_after_read($self->user_prog(),
			   grep(!defined($_->calc_exp()), $self->main()->input()->input_fields()->to_array())); #TODO: remove -- now done in parse
			$self->catalogue()->code_segments()->input_verify()->new(
				main => $self->main(), 
				user_prog => $self->user_prog(), 
				dataset => $self->main()->input()->input_dataset(),
				input_fields => $self->main()->input()->input_fields(), 
			)->prepare()
				if ($self->main()->input()->input_dataset()->verify());
	}
	sub prepare_finish
	{
		my $self = shift;
		$self->user_prog()->close_block(); # main while loop

		if ($self->main()->user_sections()->group_by()->size()) {
			my %ts_set;
			if (!$self->main()->config()->hash()) {
				$ts_set{$self->main()->output()->output_dataset()->arr_vname()}++ 
					if ($self->main()->output()->output_fields()->size());
				map($ts_set{$_->dataset()->arr_vname()}++, $self->main()->output()->transfer_fields()->to_array());
				if (keys(%ts_set)) {
					$self->user_prog()->code("if (" . join(' || ', map("\@$_", keys(%ts_set))) . ")");
					$self->user_prog()->open_block();
				}
			}
			$self->catalogue()->code_segments()->output()->new(
				main => $self->main(), user_prog => $self->user_prog())->prepare();
			$self->user_prog()->close_block() if (!$self->main()->config()->hash() && keys(%ts_set));
		}
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::InputJoin; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'input_join');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		my $r=0;
		foreach my $j ($self->main()->user_joins()->to_array()) 
		{
			$self->user_prog()->comment(" +++ Input Join (@{[ $j->input_dataset()->datasource()->fdname() ]}) Segment +++");
			$self->user_prog()->code("if");
			$self->user_prog()->open_block("(");
				$self->user_prog()->code("!defined(@{[ $j->key_field_ref()->getvar() ]})");
				$self->user_prog()->code("|| "
					. $j->join_field_ref()->getvar()
					. ($j->key_type() =~ /^(numeric|amount)$/ ? ' > ' : ' gt ')
					. $j->key_field_ref()->getvar() 
				);
			$self->user_prog()->close_block();
			$self->user_prog()->open_block();
				$j->input_dataset()->code_prepare($self->user_prog());
				$self->user_prog()->newline_off();
				$self->user_prog()->code("while (");
				$j->input_dataset()->code_read($self->user_prog(), 
				   grep(!defined($_->calc_exp()), $j->input_fields()->to_array()));
				$self->user_prog()->newline_on();
				$self->user_prog()->code(")");
				$self->user_prog()->open_block();
					$j->input_dataset()->code_after_read($self->user_prog(),
					   grep(!defined($_->calc_exp()), $j->input_fields()->to_array())); #TODO: remove -- now done in parse
					$self->user_prog()->code("++\$@{[ $j->input_dataset()->count_vname() ]};")
						if ($self->main()->config()->verbose());
					$self->catalogue()->code_segments()->input_verify()->new(
						main => $self->main(), 
						user_prog => $self->user_prog(), 
						dataset => $j->input_dataset(),
						input_fields => $j->input_fields(), 
					)->prepare()
						if ($j->input_dataset()->verify());
					$self->user_prog()->code("last if (@{[ $j->join_field_ref()->getvar() ]} le @{[ $j->key_field_ref()->getvar() ]});");
				$self->user_prog()->close_block();
			$self->user_prog()->close_block();
		}
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::InputVerify; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers qw( input_fields dataset ),
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'input_verify');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info("@{[ $self->dataset()->datasource()->fdname() ]} --> @{[ 
			uc($self->dataset()->verify()) ]}");
		$self->user_prog()->code("my \$fn=0;");
		$self->user_prog()->code("if");
		$self->user_prog()->open_block("(");
			$self->user_prog()->code("int(\@@{[ $self->dataset()->arr_vname() ]}) < @{[ 
				$self->main()->config()->show_synonyms() 
					? $self->dataset()->numflds_vname()
					: int(grep(!defined($_->calc_exp()), $self->input_fields()->to_array())) 
				]}");
			map
			(
				do {
					$self->user_prog()->code("|| (++\$fn!=@{[ $_->field_number() ]})");
					$self->user_prog()->over();
						$self->user_prog()->code("|| !defined(@{[ $_->getvar() ]})");
						$self->user_prog()->code("|| @{[ $_->getvar() ]} !~ /@{[ 
							$_->pequel_type()->validate() ]}/ # @{[ $_->pequel_type()->name() ]}"),
					$self->user_prog()->back();
				},
				grep(!defined($_->calc_exp()), $self->input_fields()->to_array())
			);
		$self->user_prog()->close_block();
		$self->user_prog()->open_block();
			if ($self->dataset()->verify() eq 'abort' || $self->main()->config()->verbose()) {
				$self->user_prog()->code("print STDERR \"Input record \", \$@{[ 
					$self->dataset()->count_vname() ]}, \" mismatch for @{[ $self->dataset()->datasource()->fdname() ]}->\@{[");
				$self->user_prog()->open_block("(");
					$self->user_prog()->code("'<num-fields>',");
					map
					(
						$self->user_prog()->code("'@{[ $_->name() ]}',"),
						grep(!defined($_->calc_exp()), $self->input_fields()->to_array())
					);