rsyncfileio.pm

老外写的linux下的文件备份软件

#============================================================= -*-perl-*-
#
# Rsync package
#
# DESCRIPTION
#
# AUTHOR
#   Craig Barratt  <[e-mail]>
#
# COPYRIGHT
#   Copyright (C) 2002-2007  Craig Barratt
#
#========================================================================
#
# Version 3.1.0, released 25 Nov 2007.
#
# See http://backuppc.sourceforge.net.
#
#========================================================================

package BackupPC::Xfer::RsyncFileIO;

use strict;
use File::Path;
use Encode qw/from_to/;
use BackupPC::Attrib qw(:all);
use BackupPC::View;
use BackupPC::Xfer::RsyncDigest qw(:all);
use BackupPC::PoolWrite;

use constant S_HLINK_TARGET => 0400000;    # this file is hardlink target
use constant S_IFMT         => 0170000;	   # type of file
use constant S_IFDIR        => 0040000;    # directory
use constant S_IFCHR        => 0020000;    # character special
use constant S_IFBLK        => 0060000;    # block special
use constant S_IFREG        => 0100000;    # regular
use constant S_IFLNK        => 0120000;    # symbolic link
use constant S_IFSOCK       => 0140000;    # socket
use constant S_IFIFO        => 0010000;    # fifo

use vars qw( $RsyncLibOK );

BEGIN {
    eval "use File::RsyncP::Digest";
    if ( $@ ) {
        #
        # Rsync module doesn't exist.
        #
        $RsyncLibOK = 0;
    } else {
        $RsyncLibOK = 1;
    }
};

sub new
{
    my($class, $options) = @_;

    return if ( !$RsyncLibOK );
    $options ||= {};
    my $fio = bless {
        blockSize    => 700,
        logLevel     => 0,
        digest       => File::RsyncP::Digest->new(),
        checksumSeed => 0,
	attrib	     => {},
	logHandler   => \&logHandler,
	stats        => {
	    errorCnt          => 0,
	    TotalFileCnt      => 0,
	    TotalFileSize     => 0,
	    ExistFileCnt      => 0,
	    ExistFileSize     => 0,
	    ExistFileCompSize => 0,
	},
	%$options,
    }, $class;

    $fio->{digest}->protocol($fio->{protocol_version});
    $fio->{shareM}   = $fio->{bpc}->fileNameEltMangle($fio->{share});
    $fio->{outDir}   = "$fio->{xfer}{outDir}/new/";
    $fio->{outDirSh} = "$fio->{outDir}/$fio->{shareM}/";
    $fio->{view}     = BackupPC::View->new($fio->{bpc}, $fio->{client},
					 $fio->{backups});
    $fio->{full}     = $fio->{xfer}{type} eq "full" ? 1 : 0;
    $fio->{newFilesFH} = $fio->{xfer}{newFilesFH};
    $fio->{partialNum} = undef if ( !$fio->{full} );
    return $fio;
}

#
# We publish our version to File::RsyncP.  This is so File::RsyncP
# can provide backward compatibility to older FileIO code.
#
# Versions:
#
#   undef or 1:  protocol version 26, no hardlinks
#   2:           protocol version 28, supports hardlinks
#
sub version
{
    return 2;
}

sub blockSize
{
    my($fio, $value) = @_;

    $fio->{blockSize} = $value if ( defined($value) );
    return $fio->{blockSize};
}

sub protocol_version
{
    my($fio, $value) = @_;

    if ( defined($value) ) {
        $fio->{protocol_version} = $value;
        $fio->{digest}->protocol($fio->{protocol_version});
    }
    return $fio->{protocol_version};
}

sub preserve_hard_links
{
    my($fio, $value) = @_;

    $fio->{preserve_hard_links} = $value if ( defined($value) );
    return $fio->{preserve_hard_links};
}

sub logHandlerSet
{
    my($fio, $sub) = @_;
    $fio->{logHandler} = $sub;
    BackupPC::Xfer::RsyncDigest->logHandlerSet($sub);
}

#
# Setup rsync checksum computation for the given file.
#
sub csumStart
{
    my($fio, $f, $needMD4, $defBlkSize, $phase) = @_;

    $defBlkSize ||= $fio->{blockSize};
    my $attr = $fio->attribGet($f, 1);
    $fio->{file} = $f;
    $fio->csumEnd if ( defined($fio->{csum}) );
    return -1 if ( $attr->{type} != BPC_FTYPE_FILE );

    #
    # Rsync uses short checksums on the first phase.  If the whole-file
    # checksum fails, then the file is repeated with full checksums.
    # So on phase 2 we verify the checksums if they are cached.
    #
    if ( ($phase > 0 || rand(1) < $fio->{cacheCheckProb})
            && $attr->{compress}
            && $fio->{checksumSeed} == RSYNC_CSUMSEED_CACHE ) {
        my($err, $d, $blkSize) = BackupPC::Xfer::RsyncDigest->digestStart(
                                     $attr->{fullPath}, $attr->{size}, 0,
                                     $defBlkSize, $fio->{checksumSeed},
                                     0, $attr->{compress}, 0,
                                     $fio->{protocol_version});
        my($isCached, $isInvalid) = $d->isCached;
        if ( $fio->{logLevel} >= 5 ) {
            $fio->log("$attr->{fullPath} verify; cached = $isCached,"
                    . " invalid = $isInvalid, phase = $phase");
        }
        if ( $isCached || $isInvalid ) {
            my $ret = BackupPC::Xfer::RsyncDigest->digestAdd(
                            $attr->{fullPath}, $blkSize,
                            $fio->{checksumSeed}, 1,        # verify
                            $fio->{protocol_version}
                        );
            if ( $ret != 1 ) {
                $fio->log("Bad cached digest for $attr->{fullPath} ($ret);"
                        . " fixed");
                $fio->{stats}{errorCnt}++;
            } else {
                $fio->log("$f->{name}: verified cached digest")
                                    if ( $fio->{logLevel} >= 2 );
            }
        }
        $d->digestEnd;
    }
    (my $err, $fio->{csum}, my $blkSize)
         = BackupPC::Xfer::RsyncDigest->digestStart($attr->{fullPath},
			 $attr->{size}, 0, $defBlkSize, $fio->{checksumSeed},
			 $needMD4, $attr->{compress}, 1, $fio->{protocol_version});
    if ( $err ) {
        $fio->log("Can't get rsync digests from $attr->{fullPath}"
                . " (err=$err, name=$f->{name})");
	$fio->{stats}{errorCnt}++;
        return -1;
    }
    if ( $fio->{logLevel} >= 5 ) {
        my($isCached, $invalid) = $fio->{csum}->isCached;
        $fio->log("$attr->{fullPath} cache = $isCached,"
                . " invalid = $invalid, phase = $phase");
    }
    return $blkSize;
}

sub csumGet
{
    my($fio, $num, $csumLen, $blockSize) = @_;
    my($fileData);

    $num     ||= 100;
    $csumLen ||= 16;
    return if ( !defined($fio->{csum}) );
    return $fio->{csum}->digestGet($num, $csumLen);
}

sub csumEnd
{
    my($fio) = @_;

    return if ( !defined($fio->{csum}) );
    return $fio->{csum}->digestEnd();
}

sub readStart
{
    my($fio, $f) = @_;

    my $attr = $fio->attribGet($f, 1);
    $fio->{file} = $f;
    $fio->readEnd if ( defined($fio->{fh}) );
    if ( !defined($fio->{fh} = BackupPC::FileZIO->open($attr->{fullPath},
                                           0,
                                           $attr->{compress})) ) {
        $fio->log("Can't open $attr->{fullPath} (name=$f->{name})");
	$fio->{stats}{errorCnt}++;
        return;
    }
    $fio->log("$f->{name}: opened for read") if ( $fio->{logLevel} >= 4 );
}

sub read
{
    my($fio, $num) = @_;
    my $fileData;

    $num ||= 32768;
    return if ( !defined($fio->{fh}) );
    if ( $fio->{fh}->read(\$fileData, $num) <= 0 ) {
        return $fio->readEnd;
    }
    $fio->log(sprintf("read returns %d bytes", length($fileData)))
				if ( $fio->{logLevel} >= 8 );
    return \$fileData;
}

sub readEnd
{
    my($fio) = @_;

    return if ( !defined($fio->{fh}) );
    $fio->{fh}->close;
    $fio->log("closing $fio->{file}{name})") if ( $fio->{logLevel} >= 8 );
    delete($fio->{fh});
    return;
}

sub checksumSeed
{
    my($fio, $checksumSeed) = @_;

    $fio->{checksumSeed} = $checksumSeed;
    $fio->log("Checksum caching enabled (checksumSeed = $checksumSeed)")
        if ( $fio->{logLevel} >= 1 && $checksumSeed == RSYNC_CSUMSEED_CACHE );
    $fio->log("Checksum seed is $checksumSeed")
        if ( $fio->{logLevel} >= 2 && $checksumSeed != RSYNC_CSUMSEED_CACHE );
}

sub dirs
{
    my($fio, $localDir, $remoteDir) = @_;

    $fio->{localDir}  = $localDir;
    $fio->{remoteDir} = $remoteDir;
}

sub viewCacheDir
{
    my($fio, $share, $dir) = @_;
    my $shareM;

    #$fio->log("viewCacheDir($share, $dir)");
    if ( !defined($share) ) {
	$share  = $fio->{share};
	$shareM = $fio->{shareM};
    } else {
	$shareM = $fio->{bpc}->fileNameEltMangle($share);
    }
    $shareM = "$shareM/$dir" if ( $dir ne "" );
    return if ( defined($fio->{viewCache}{$shareM}) );
    #
    # purge old cache entries (ie: those that don't match the
    # first part of $dir).
    #
    foreach my $d ( keys(%{$fio->{viewCache}}) ) {
	delete($fio->{viewCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
    }
    #
    # fetch new directory attributes
    #
    $fio->{viewCache}{$shareM}
		= $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
    #
    # also cache partial backup attrib data too
    #
    if ( defined($fio->{partialNum}) ) {
        foreach my $d ( keys(%{$fio->{partialCache}}) ) {
            delete($fio->{partialCache}{$d}) if ( $shareM !~ m{^\Q$d/} );
        }
        $fio->{partialCache}{$shareM}
                    = $fio->{view}->dirAttrib($fio->{partialNum}, $share, $dir);
    }
}

sub attribGetWhere
{
    my($fio, $f, $noCache, $fname) = @_;
    my($dir, $share, $shareM, $partial, $attr);

    if ( !defined($fname) ) {
        $fname = $f->{name};
        $fname = "$fio->{xfer}{pathHdrSrc}/$fname"
		       if ( defined($fio->{xfer}{pathHdrSrc}) );
    }
    $fname =~ s{//+}{/}g;
    if ( $fname =~ m{(.*)/(.*)}s ) {
	$shareM = $fio->{shareM};
	$dir = $1;
	$fname = $2;
    } elsif ( $fname ne "." ) {
	$shareM = $fio->{shareM};
	$dir = "";
    } else {
	$share = "";
	$shareM = "";
	$dir = "";
	$fname = $fio->{share};
    }
    $shareM .= "/$dir" if ( $dir ne "" );

    if ( $noCache ) {
        $share  = $fio->{share} if ( !defined($share) );
        my $dirAttr = $fio->{view}->dirAttrib($fio->{viewNum}, $share, $dir);
        $attr = $dirAttr->{$fname};
    } else {
        $fio->viewCacheDir($share, $dir);
        if ( defined($attr = $fio->{viewCache}{$shareM}{$fname}) ) {
            $partial = 0;
        } elsif ( defined($attr = $fio->{partialCache}{$shareM}{$fname}) ) {
            $partial = 1;
        } else {
            return;