table.pm

普通的ETL工具

	use base qw(ETL::Pequel3::Type::Table::Local);
	use Class::STL::ClassMembers qw( input_dataset ),
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => "The data for this table type is loaded from an external I<dataset>. This table type can be loaded I<dynamically> or I<statically>.\n\n\nThe advantage of the F<static> type over the F<dynamic> type is (1) there is no wait, at the start of execution, for the table data to load; (2) the table-data is combined with the generated program so will execute without any dependence on other external files. The disadvantages with F<static> is (1) the program size, especially with large data volumes, and (2) the program will need to be re-generated whenever the data needs to be refreshed.\n\n\nF<Dynanic Table>\n\nThe data for a dynamic type table is loaded at run-time, that is, when the I<generated> program executes.\n\n\nF<Static Table>\n\nWith static tables the data is pre-loaded during the code-generation resulting with the table-data embedded within the generated program (as a I<hash> variable)."),
		Class::STL::ClassMembers::DataMember->new(name => 'name', default => 'external');
	use Class::STL::ClassMembers::Constructor;
	sub factory
	{
		my $self = shift;
		my %p = @_; # table_type, static, dataset_spec, ...
		return 
		(
			defined($p{input_dataset}) 
			&& 
			(
				$p{table_type} eq $self->name()
				|| 
				(
					$p{table_type} eq 'external'
					&& 
					( 
						($self->isa('ETL::Pequel3::Type::Table::ExternalStatic') && $p{static})
						|| 
						(!$self->isa('ETL::Pequel3::Type::Table::ExternalStatic') && !$p{static})
					)
				)
			)
		) 
		? $self->new(%p) : 0;
	}
	sub prepare
	{
		my $self = shift;
		my $items = shift;

		my $key_type = $self->key_type() || 'string';
		$key_type = $self->pequel_types()->exists($key_type)
			|| $self->err()->user_error(10210, "Unkown pequel-type '@{[ $key_type ]}' for table '@{[ $self->name() ]}' key_field");
	
		$self->err()->user_error(10213, "Key field not defined for table '@{[ $self->name() ]}'")
			unless (defined($self->key_field()));

		# Disable discard_header because sort will shift it:
		$self->input_dataset()->discard_header(0) if ($self->input_dataset()->can('discard_header'));

		$self->input_dataset()->map_output(field_map => $self->field_map(), key_field => $self->key_field());
		$self->err()->user_error(10214, "Table @{[ $self->name() ]}->key field '@{[ 
				$self->key_field() ]}' does not exist in source '@{[ 
				$self->input_dataset()->datasource()->target_name()
			]}'")
			unless (grep($self->key_field() eq $_->name(), $self->fields()->to_array()));
		$self->attach_sorter($key_type);
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::Table::ExternalStatic; # preload table data into generated code.
	use base qw(ETL::Pequel3::Type::Table::External::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => "The data for this table type is loaded from an external I<dataset>. With static tables the data is pre-loaded during the code-generation resulting with the table-data embedded within the generated program (as a I<hash> variable)."),
		Class::STL::ClassMembers::DataMember->new(name => 'name', default => 'external_static');
	use Class::STL::ClassMembers::Constructor;
	sub code_load_table
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program->new();
		our %__loaded;

		my $preload_function = "preload@{[ $self->tname() ]}";

		# Prevent re-evaling preload function when same static table declared more 
		# than once (as in some tests)
		return $self->SUPER::code_load_table($c) if (++$__loaded{$preload_function} > 1);

		my $ctmp = ETL::Pequel3::CodeStyler::Program->new();
		$ctmp->open_block();
		$ctmp->code("package " . $self->package_name() . ";");
		$ctmp->code("sub $preload_function");
		$ctmp->open_block();
		$ctmp->code("my \$self = shift;");
		$ctmp->code("my \$data = shift;");
		$self->input_dataset()->code_init($ctmp);
		$self->catalogue()->code_segments()->verbose_msg()->new(
			target_name => $self->properties()->script_name(),
			user_prog => $ctmp,
			msg_text => "Pre-loading static table @{[ $self->name() ]} from @{[ 
				$self->input_dataset()->datasource()->target_name() ]}..."
		)->prepare() if ($self->configuration()->verbose()); 
		$self->input_dataset()->code_open($ctmp);
		$self->input_dataset()->code_prepare($ctmp);
		$ctmp->newline_off();
		$ctmp->code("while (");
		$self->input_dataset()->code_read($ctmp, $self->fields()->to_array());
		$ctmp->newline_on();
		$ctmp->code(")");
		$ctmp->open_block();
			$self->input_dataset()->code_after_read($ctmp, $self->fields()->to_array());
			$ctmp->newline_off();
			$ctmp->code("\$data->push_back(\$data->factory(");
			$ctmp->code("key => \$@{[ $self->input_dataset()->arr_vname() ]}\[@{[ 
				$self->fields()->exists($self->key_field())->ds_column() -1 ]}],");
			$ctmp->code("value => [ ");
			$self->fields()->size() == 0
				? $ctmp->code("1")
				: $ctmp->code("\@@{[ $self->input_dataset()->arr_vname() ]}\[ 
					@{[ join(',', map($_->ds_column() -1, $self->fields()->to_array())) ]} ] "); # includes key as well
			$ctmp->code("]");
			$ctmp->code(")"); # close factory
			$ctmp->newline_on();
			$ctmp->code(");"); # close push_back
			$self->catalogue()->code_segments()->verbose_msg()->new(
				target_name => $self->properties()->script_name(),
				user_prog => $ctmp,
				msg_text => "Table @{[ $self->name ]} \$. records...", 
				cond => "if (\$. % 100000 == 0)"
			)->prepare() if ($self->configuration()->verbose()); 
		$ctmp->close_block(); # while
		$self->catalogue()->code_segments()->verbose_msg()->new(
			target_name => $self->properties()->script_name(),
			user_prog => $ctmp,
			msg_text => "Table @{[ $self->name() ]} pre-loaded \$. records."
		)->prepare() if ($self->configuration()->verbose()); 
		$self->input_dataset()->code_close($ctmp);
		$ctmp->close_block(); # sub
		$ctmp->close_block(); # package
		if ($self->input_dataset()->datasource()->datasource_name() eq 'pequel') {
			my $pname = "Pequel::@{[ $self->input_dataset()->datasource()->target_name() ]}\::execute";
			undef(&{$pname});
			my $sub_code = ETL::Pequel3::Generator::SubProgram->new(
				main => $self->input_dataset()->datasource()->pequel_ref());
			$sub_code->build();
			$ctmp->add($sub_code->user_prog());
		}
		$ctmp->prepare();
		$self->err()->trace_msg(10, $ctmp->print());
		$ctmp->eval();
		$self->data()->clear();
		$self->$preload_function($self->data());
		return $self->SUPER::code_load_table($c);
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::Table::ExternalDynamic; # load table at runtime.
	use base qw(ETL::Pequel3::Type::Table::External::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => "The data for this table type is loaded from an external I<dataset>. The data for a dynamic type table is loaded at run-time, that is, when the I<generated> program executes."),
		Class::STL::ClassMembers::DataMember->new(name => 'name', default => 'external_dynamic');
	use Class::STL::ClassMembers::Constructor;
	sub code_load_table
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program->new(); 
		$c->divider();
		$self->code_display_info($c, "@{[ __PACKAGE__ ]}");
		$c->open_block("{");
		$c->code("package " . $self->package_name() . ";");
		$c->code("sub @{[ $self->load_function_name() ]}");
		$c->open_block("{");
		$self->input_dataset()->code_init($c);
		$self->catalogue()->code_segments()->verbose_msg()->new(
			target_name => $self->properties()->script_name(),
			user_prog => $c,
			msg_text => "Loading table @{[ $self->name() ]} from @{[ 
				$self->input_dataset()->datasource()->target_name() 
			]}..."
		)->prepare() if ($self->configuration()->verbose()); 
		$self->input_dataset()->code_open($c);
		$self->input_dataset()->code_prepare($c);
		$c->code("my \%@{[ $self->tname() ]};");
		$c->newline_off();
		$c->code("while (");
		$self->input_dataset()->code_read($c, $self->fields()->to_array());
		$c->newline_on();
		$c->code(")");
		$c->open_block("{");
			$self->input_dataset()->code_after_read($c, $self->fields()->to_array());
			$c->newline_off();
			$c->code("\$@{[ $self->tname() ]}\{\$@{[ 
					$self->input_dataset()->arr_vname() 
				]}\[@{[ 
					$self->fields()->exists($self->key_field())->ds_column() -1 
				]}]} = ");

			$c->newline_on();
			($self->fields()->size() == 0)
				? $c->code("'1',")
				: $c->code("[ \@@{[ $self->input_dataset()->arr_vname() ]}\[ @{[ 
					join(',', map($_->ds_column() -1, $self->fields()->to_array())) 
				]} ] ];");

			$self->catalogue()->code_segments()->verbose_msg()->new(
				target_name => $self->properties()->script_name(),
				user_prog => $c,
				msg_text => "Table @{[ $self->name ]} \$. records...", 
				cond => "if (\$. % 100000 == 0)"
			)->prepare() if ($self->configuration()->verbose()); 
		$c->close_block();
		$self->catalogue()->code_segments()->verbose_msg()->new(
			target_name => $self->properties()->script_name(),
			user_prog => $c,
			msg_text => "Table @{[ $self->name() ]} loaded \$. records."
		)->prepare() if ($self->configuration()->verbose()); 
		$self->input_dataset()->code_close($c);
		$c->code('return \%' . $self->tname() . ';');
		$c->close_block();
		$c->close_block();
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::Table::Dbi::Abstract;
	use base qw(ETL::Pequel3::Type::Table::External::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'name', default => 'dbi'),
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => "The data for this table type is loaded from an external F<dbi> I<dataset>. This table type can be loaded I<dynamically> or I<statically>. Please refer to the description for F<external> for further information regarding F<static> and F<dynamic>."),
		Class::STL::ClassMembers::DataMember->new(name => 'fieldset_type', default => 'table_fields_dbi');
	use Class::STL::ClassMembers::Constructor;
	sub factory
	{
		my $self = shift;
		my %p = @_; # table_type, static, dataset_spec, ...
		return 
		(
			defined($p{input_dataset}) 
			&& 
			(
				$p{table_type} eq $self->name()
				|| 
				(
					$p{table_type} eq 'dbi'
					&& 
					( 
						($self->isa('ETL::Pequel3::Type::Table::ExternalStatic') && $p{static})
						|| 
						(!$self->isa('ETL::Pequel3::Type::Table::ExternalStatic') && !$p{static})
					)
				)
			)
		) 
		? $self->new(%p) : 0;
	}
	sub attach_sorter
	{
		my $self = shift;
		my $key_type = shift;
#>		my $key_field = shift;
#>		$self->input_dataset()->select_order_by($self->key_field());
	}
}
# ----------------------------------------------------------------------------------------------------
{	
	package ETL::Pequel3::Type::Table::DbiStatic;
	use base qw(ETL::Pequel3::Type::Table::Dbi::Abstract);
	use base qw(ETL::Pequel3::Type::Table::ExternalStatic);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'name', default => 'dbi_static'),
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => "The data for this table type is loaded from an external F<dbi> I<dataset>. With static tables the data is pre-loaded during the code-generation resulting with the table-data embedded within the generated program (as a I<hash> variable)."),
		Class::STL::ClassMembers::DataMember->new(name => 'fieldset_type', default => 'table_fields');
	use Class::STL::ClassMembers::Constructor;
	sub code_load_table
	{
		my $self = shift;
		return $self->ETL::Pequel3::Type::Table::ExternalStatic::code_load_table(@_);
	}
}