datasource.pm

普通的ETL工具

		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		defined($self->record_delimiter()) && $self->record_delimiter() ne ''
			? $c->code("local(\$\\)=\"@{[ $self->record_delimiter() ]}\";")
			: $c->code("local(\$\\)=undef;");
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Dbi;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'dbi');
	use Class::STL::ClassMembers::Constructor;
	sub _code_prepare
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_close
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_open
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::SqlPlus;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers qw( filter_cmd ),
		qw( 
			dsn 
			user 
			password 
			table_name 
			select_override 
			select_where_condition 
			select_fields
			select_order_by
			select_group_by
		),
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'sqlplus');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::External;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'description', default => 'Specify an external command to send (pipe) the incoming datastream.'),
		ETL::Pequel3::Type::DataMember::User->new(name => 'filter_cmd'),
		ETL::Pequel3::Type::DataMember::User->new(name => 'target_name'),
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'external');
	use Class::STL::ClassMembers::Constructor;
	sub new_extra
	{
		my $self = shift;
		my $path_delim = '/';
#<		$self->ds_vname("_DS@{[ $self->fdname() ]}") if (defined($self->fdname()));
		return $self;
	}
	sub _code_init 
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		$c->code("my \$@{[ $self->ds_vname() ]} = '@{[ $self->target_name() ]}';");
		return $c;
	}
	# target_name contains the external command...
	# my _DS_EXT1 = "perl script.pl arg arg arg"
	# open(FD, "$_DS_EXT1 2>/dev/null |");
	# open(FD, "$_DS_EXT1 2>/dev/null | filter_cmd 2>/dev/null |");
	# open(FD, "$_DS_EXT1 2>/dev/null | filter_cmd 2>/dev/null | sort arg arg 2>/dev/null |");
	# open(FD, "$_DS_EXT1 2>/dev/null | sort arg arg 2>/dev/null |");
	#
	sub _code_open
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		$c->newline_off();
		$c->code("open(@{[ $self->fdname() ]}, \"@{[ $self->ds_vname() ]} 2>/dev/null |");
		$c->code(" @{[ $self->filter_cmd() ]} 2>/dev/null |") if (defined($self->filter_cmd()));
		$c->code(" @{[ $self->sorter()->code_open()->raw() ]} 2>/dev/null |") if (defined($self->sorter()));
		$c->newline_on();
		$c->code('")');
		$c->over();
		$c->code(" || die(\"Unable to open external datasource '\$@{[ $self->ds_vname() ]}'\"");
		$c->over();
		$c->code(". \" for '@{[ $self->fdname() ]}'\");");
		$c->back();
		$c->back();
		return $c;
	}
	sub _code_close
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		$c->code("close(@{[ $self->fdname() ]});");
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::External;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'external');
	use Class::STL::ClassMembers::Constructor;
	#TODO
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Socket;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'socket');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Socket;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'socket');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Fifo;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'fifo');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Fifo;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'fifo');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::MsgQueue;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'msg_queue');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::MsgQueue;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'msg_queue');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Void;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'void'),
		Class::STL::ClassMembers::DataMember->new(name => 'target_name', default => 'VOID');
	use Class::STL::ClassMembers::Constructor;
	sub _code_prepare
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_close
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_open
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Void;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'datasource_name', default => 'void'),
		Class::STL::ClassMembers::DataMember->new(name => 'target_name', default => 'VOID');
	use Class::STL::ClassMembers::Constructor;
	sub _code_prepare
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_close
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
	sub _code_open
	{
		my $self = shift;
		my $c = shift || ETL::Pequel3::CodeStyler::Program::Perl->new();
		return $c;
	}
}
# ----------------------------------------------------------------------------------------------------
# CATALOGUE
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Catalogue::Abstract;
	use base qw(ETL::Pequel3::Type::Catalogue);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'target_mem_name', default => 'datasource_name'),
		Class::STL::ClassMembers::DataMember->new(name => 'element_type', default => 'ETL::Pequel3::Type::DataSource::Abstract');
	use Class::STL::ClassMembers::Constructor;
	sub catalogue # --> to_xml
	{
		my $self = shift;
		my $xml_node = shift;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Catalogue;
	use base qw(ETL::Pequel3::Type::DataSource::Catalogue::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'catalogue_name', default => 'input_datasources');
	use Class::STL::ClassMembers::SingletonConstructor;
	sub new_extra
	{
		my $self = shift;
		use ETL::Pequel3::Type::DataSource::Pequel;
		$self->push_back(
			ETL::Pequel3::Type::DataSource::Input::File->_new(),
			ETL::Pequel3::Type::DataSource::Input::Pipe->_new(),
			ETL::Pequel3::Type::DataSource::Input::Pequel->_new(), # in DataSource/Pequel.pm
			ETL::Pequel3::Type::DataSource::Input::External->_new(),
			ETL::Pequel3::Type::DataSource::Input::Dbi->_new(),
#>			ETL::Pequel3::Type::DataSource::Input::Socket->_new(),
			ETL::Pequel3::Type::DataSource::Input::Void->_new(),
#>			ETL::Pequel3::Type::DataSource::Input::Fifo->_new(),
#>			ETL::Pequel3::Type::DataSource::Input::MsgQueue->_new(),
		);
		return $self;
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Catalogue;
	use base qw(ETL::Pequel3::Type::DataSource::Catalogue::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'catalogue_name', default => 'output_datasources');
	use Class::STL::ClassMembers::SingletonConstructor;
	sub new_extra
	{
		my $self = shift;
		use ETL::Pequel3::Type::DataSource::Pequel;
		$self->push_back(
			ETL::Pequel3::Type::DataSource::Output::File->_new(),
			ETL::Pequel3::Type::DataSource::Output::Pipe->_new(),
			ETL::Pequel3::Type::DataSource::Output::Pequel->_new(), # in DataSource/Pequel.pm
#>			ETL::Pequel3::Type::DataSource::Output::Socket->_new(),
			ETL::Pequel3::Type::DataSource::Output::Void->_new(),
#>			ETL::Pequel3::Type::DataSource::Output::Fifo->_new(),
#>			ETL::Pequel3::Type::DataSource::Output::MsgQueue->_new(),
		);
		return $self;
	}
}
# ----------------------------------------------------------------------------------------------------
# FACTORY
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Factory::Abstract;
	use ETL::Pequel3::Type::Factory;
	use base qw(ETL::Pequel3::Type::Factory::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'factory_name', default => 'datasourse'),
		Class::STL::ClassMembers::DataMember->new(name => 'target_mem_name', default => 'datasource_name'),
		Class::STL::ClassMembers::DataMember->new(name => 'element_type', default => 'ETL::Pequel3::Type::DataSource::Abstract');
	use Class::STL::ClassMembers::Constructor;
	sub factory
	{
		my $self = shift;
		use Carp qw(confess);
		my %p = @_;
		# stdin
		# stdout
		# void
		# <target_name>
		# pequel(target_name=...)
		# pequel(<pequel_name>)
		# file(<filename>)
		# socket(args...)

		my $type = exists($p{datasource_spec}) ? $p{datasource_spec} : 'void';
		if ($type =~ /\s*:?\s*(\w+)\s*\((.*?)\)\s*$/) 
		{
			$type = $1; # must be datasource-type
			$self->err()->user_error(10801, "Unknown datasource type '$type' @{[ 
				exists($p{datasource_spec}) ? 'in spec ' . $p{datasource_spec} : '' ]}")
				unless ($self->catalogue()->exists(lc($type)));
			my $args = $2;
			use ETL::Pequel3::Parser;
			my @args = ETL::Pequel3::Parser->new(pequel_ref => $p{pequel_ref})->extract_args($args);
			%p = (int(@args) == 1 ? ('target_name', @args) : @args);
		}
		return $self->catalogue()->exists(lc($type))
			? $self->catalogue()->exists(lc($type))->new(@_, %p)
			: defined($p{datasource_type})
				? $self->catalogue()->exists($p{datasource_type})->new('target_name' => $type, @_, %p)
				: $self->catalogue()->exists('file')->new('target_name' => $type, @_, %p);
	}
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Factory::Abstract;
	use base qw(ETL::Pequel3::Type::DataSource::Factory::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'catalogue_type', default => 'input_datasources');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Factory::Abstract;
	use base qw(ETL::Pequel3::Type::DataSource::Factory::Abstract);
	use Class::STL::ClassMembers
		Class::STL::ClassMembers::DataMember->new(name => 'catalogue_type', default => 'output_datasources');
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Factory::Local;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Factory::Abstract);
	use Class::STL::ClassMembers;
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Factory::Local;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Factory::Abstract);
	use Class::STL::ClassMembers;
	use Class::STL::ClassMembers::Constructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Input::Factory::Global;
	use base qw(ETL::Pequel3::Type::DataSource::Input::Factory::Abstract);
	use Class::STL::ClassMembers;
	use Class::STL::ClassMembers::SingletonConstructor;
}
# ----------------------------------------------------------------------------------------------------
{
	package ETL::Pequel3::Type::DataSource::Output::Factory::Global;
	use base qw(ETL::Pequel3::Type::DataSource::Output::Factory::Abstract);
	use Class::STL::ClassMembers;
	use Class::STL::ClassMembers::SingletonConstructor;
}
# ----------------------------------------------------------------------------------------------------
1;