generator.pm

普通的ETL工具

		$self->user_prog()->divider();
		$self->user_prog()->comment(" vim: syntax=perl ts=@{[ 
			$self->main()->config()->tab_type() eq 'hard' 
				? '1' :  $self->main()->config()->tab_size() 
			]} sw=@{[ $self->main()->config()->tab_size() ]}");
		$self->user_prog()->divider();
		$self->user_prog()->comment(" Generated By: Pequel3 @{[ $self->properties()->version() ]}");
		$self->user_prog()->comment("             : http://sourceforge.net/projects/pequel/");
		$self->user_prog()->comment(" Created On  : @{[ '' . localtime() ]}");
		$self->user_prog()->comment(" Perl Version: @{[ $^X . ' ' . sprintf(qq{%vd}, $^V) . ' on ' . $^O ]}");
		$self->user_prog()->comment(" For         : @{[ getlogin() || '' ]}");
		$self->user_prog()->divider();
		$self->user_prog()->code("use strict;");
		$self->user_prog()->code("use warnings;");
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::Divider; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'divider');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->user_prog()->divider();
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::EndOfProgram; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'end_of_program');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->user_prog()->divider();
		$self->user_prog()->comment(" ++++ END OF PROGRAM -- @{[ $self->main()->script_filename() ]} ++++");
		$self->user_prog()->divider();
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::CallLoadTable; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'call_load_table');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		map
		(
			$_->code_start($self->user_prog()),
			grep(defined($_), $self->main()->user_tables()->to_array())
		);
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::CallExecute; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'call_execute');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		$self->user_prog()->code("&@{[ $self->package_name() ]}::execute();");
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::Package; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'package');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->user_prog()->open_block();
		$self->user_prog()->code("package @{[ $self->package_name() ]};");
	}
	sub prepare_finish
	{
		my $self = shift;
		$self->user_prog()->close_block();
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::HeaderInfo; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'header_info');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->user_prog()->divider();
		$self->user_prog()->comment(" Script Name : @{[ $self->main()->pequel_name() ]}");
		$self->user_prog()->divider();
		$self->user_prog()->comment(" Configuration:");
		foreach 
		(
			grep
			(
				$_->user_program_info() != 0 && defined($_->value()),
					grep($_->global(), $self->properties()->to_array()), 
					$self->main()->config()->to_array()
			)
		) 
		{
			$self->user_prog()->comment
			(
				" "
				. $_->name()
				. " -- "
				. ($_->value() =~ /^(0|1)$/ ? ($_->value() == 0 ? 'NO' : 'YES') : $_->value())
				. " -- "
				. $_->description()
				. ($_->global() ? ' -- (global)' : '')
			);
		}
		$self->user_prog()->divider();
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::Declaration;  # all local
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'declaration');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		foreach 
		(
			grep
			(
				defined($_),
				(
					$self->main()->input()->input_fields(),
					$self->main()->output()->output_fields(),
					map($_->input_fields(), $self->main()->user_joins()->to_array()),
					map($_->fields(), $self->main()->user_tables()->to_array()),
					map($_->output_fields(), 
						map($_->reason_codes_list()->to_array(), 
							$self->main()->user_sections()->reject_record_on_input()->to_array())),
					map($_->output_fields(), 
						map($_->reason_codes_list()->to_array(), 
							$self->main()->user_sections()->reject_record_on_output()->to_array())),
				)
			)
		)
		{
			$_->code_decl($self->user_prog());
		}
		$self->user_prog()->code("use constant _VERBOSE => int @{[ $self->main()->config()->verbose() ]};")
			if ($self->main()->config()->verbose());

		$self->user_prog()->code("use constant @{[ $self->main()->input()->input_dataset()->numflds_vname() ]} => int "
		   . int(grep(!defined($_->calc_exp()), $self->main()->input()->input_fields()->to_array())) 
		   . ";");

		foreach my $j ($self->main()->user_joins()->to_array()) 
		{
			$self->user_prog()->code("use constant @{[ $j->input_dataset()->numflds_vname() ]} => int "
			   . int(grep(!defined($_->calc_exp()), $j->input_fields()->to_array())) 
			   . ";");
		}
	}
}
# ----------------------------------------------------------------------------------------------------
{	
	package ETL::Pequel3::Generator::Segment::InitGlobal;
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'init_global');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		# Only wan't to the ds_var code from datasource which is relevant to the open command:
		map($_->datasource()->_code_init($self->user_prog()), $self->main()->shared_datasets()->to_array());
	}
}	
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::OpenGlobal; 
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'open_global');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		foreach ($self->main()->shared_datasets()->to_array())
		{
			$self->user_prog()->comment("");
			$self->user_prog()->comment(" ---> @{[ 
					ucfirst($_->datasource()->io_name()) 
				]}: @{[ $_->datasource()->fdname() ]} (@{[ 
				$_->name() ]}:@{[ $_->datasource()->datasource_name() ]})");
			$self->user_prog()->comment("");
			$_->code_open($self->user_prog());
		}
		foreach 
		(
			grep
			(
				$_->can('print_header') && $_->print_header(),
				(
					$self->main()->shared_datasets()->to_array(), 
				)
			)
		) 
		{
			$self->catalogue()->code_segments()->print_output_header()->new(
				main => $self->main(), user_prog => $self->user_prog(), dataset => $_)
				->prepare();
		}
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::CloseGlobal;
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'close_global');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		$self->program_info();
		map($_->code_close($self->user_prog()), reverse $self->main()->shared_datasets()->to_array());
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Generator::Segment::InitLocal; #-->Initialisation
	use base qw(ETL::Pequel3::Generator::Segment::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'segment_name', default => 'init_local');
	use Class::STL::ClassMembers::Constructor; 
	sub prepare_start
	{
		my $self = shift;
		my %__shared_ds;
		$self->program_info();

		if ($self->main->macro_use_list()->size() > 1) {
			stl::stable_sort($self->main->macro_use_list()->begin(), $self->main->macro_use_list()->end());
			stl::unique($self->main->macro_use_list()->begin(), $self->main->macro_use_list()->end());
		}
		foreach ($self->main->macro_use_list()->to_array()) {
			$_->code_init_global($self->user_prog());
			#TODO:code_init()
		}

		foreach 
		(
			map($_->aggregate_type(), grep(defined($_->aggregate_type()), $self->main()->output()->output_fields()->to_array())),
			grep
			(
				defined($_) && (!$_->can('shared') || !$_->shared() || ++$__shared_ds{$_->arr_vname()} <= 1),
				(
#<					$self->main()->user_datasets()->to_array() # !! contains repeated ds's
					map($_->input_dataset(), $self->main()->user_joins()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_input()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->copy_record_on_output()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_input()->to_array()),
					map($_->output_dataset(), $self->main()->user_sections()->divert_record_on_output()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_input()->to_array()),
					map($_->dataset(), $self->main()->user_sections()->reject_record_on_output()->to_array()),
					$self->main()->input()->input_dataset(),
					$self->main()->output()->transfer_dataset(),
					map($_->transfer_dataset(), $self->main()->user_joins()->to_array()),
					$self->main()->output()->output_dataset(),
				)
			)
		)
		{
			$_->code_init($self->user_prog()); # arr_vname
		}
		if ($self->main()->user_sections()->group_by()->size()) {
			$self->user_prog()->comment("");
			$self->user_prog()->comment(" ---> Break control grouping fields");
			$self->user_prog()->comment("");
			foreach ($self->main()->group_by()->input_fields()->to_array()) {
				$self->user_prog()->code("my \$key@{[ $_->tname() ]};");
				$self->user_prog()->code("my \$previous_key@{[ $_->tname() ]} = undef;");
			}
			if ($self->main()->config()->hash()) {
				$self->user_prog()->comment("");
				$self->user_prog()->comment(" ---> Configuration >> hash <<");
				$self->user_prog()->comment("");
				$self->user_prog()->code("my \$_HKEY;");
			}
		}
		if ($self->main()->user_sections()->dedup_on_input()->size()) {
			foreach ($self->main()->dedup_on_input()->fields()->to_array()) {
				$self->user_prog()->code("my \$previous_dedup@{[ $_->input_field()->tname() ]} = undef;");
			}
		}
		if ($self->main()->user_sections()->dedup_on_output()->size()) {
			foreach ($self->main()->dedup_on_output()->fields()->to_array()) {
				$self->user_prog()->code("my \$previous_dedup@{[ $_->output_field()->tname() ]} = undef;");
			}
		}
		if ($self->main()->config()->verbose() && $self->main()->config()->statistics()) {
			$self->user_prog()->comment("");