rsyncfileio.pm

老外写的linux下的文件备份软件

# Once as the file list is processed, and again at
# the end.  BackupPC does them as it goes (since it is
# just saving the hardlink info and not actually making
# hardlinks).
#
sub makeHardLink
{
    my($fio, $f, $end) = @_;

    return if ( $end );
    return $fio->makeSpecial($f) if ( !$f->{hlink_self} );
}

sub unlink
{
    my($fio, $path) = @_;
    
    $fio->log("Unexpected call BackupPC::Xfer::RsyncFileIO->unlink($path)"); 
}

#
# Default log handler
#
sub logHandler
{
    my($str) = @_;

    print(STDERR $str, "\n");
}

#
# Handle one or more log messages
#
sub log
{
    my($fio, @logStr) = @_;

    foreach my $str ( @logStr ) {
        next if ( $str eq "" );
        $fio->{logHandler}($str);
    }
}

#
# Generate a log file message for a completed file
#
sub logFileAction
{
    my($fio, $action, $f) = @_;
    my $owner = "$f->{uid}/$f->{gid}";
    my $type  = (("", "p", "c", "", "d", "", "b", "", "", "", "l", "", "s"))
		    [($f->{mode} & S_IFMT) >> 12];
    my $name = $f->{name};

    if ( ($f->{mode} & S_IFMT) == S_IFLNK ) {
        $name .= " -> $f->{link}";
    } elsif ( ($f->{mode} & S_IFMT) == S_IFREG
            && defined($f->{hlink}) && !$f->{hlink_self} ) {
        $name .= " -> $f->{hlink}";
    }
    $name =~ s/\n/\\n/g;

    $fio->log(sprintf("  %-6s %1s%4o %9s %11.0f %s",
				$action,
				$type,
				$f->{mode} & 07777,
				$owner,
				$f->{size},
                                $name));
}

#
# If there is a partial and we are doing a full, we do an incremental
# against the partial and a full against the rest.  This subroutine
# is how we tell File::RsyncP which files to ignore attributes on
# (ie: against the partial dump we do consider the attributes, but
# otherwise we ignore attributes).
#
sub ignoreAttrOnFile
{
    my($fio, $f) = @_;

    return if ( !defined($fio->{partialNum}) );
    my($attr, $isPartial) = $fio->attribGetWhere($f);
    $fio->log("$f->{name}: just checking attributes from partial")
                                if ( $isPartial && $fio->{logLevel} >= 5 );
    return !$isPartial;
}

#
# This is called by File::RsyncP when a file is skipped because the
# attributes match.
#
sub attrSkippedFile
{
    my($fio, $f, $attr) = @_;

    #
    # Unless this is a partial, this is normal so ignore it.
    #
    return if ( !defined($fio->{partialNum}) );

    $fio->log("$f->{name}: skipped in partial; adding link")
                                    if ( $fio->{logLevel} >= 5 );
    $fio->{rxLocalAttr} = $attr;
    $fio->{rxFile} = $f;
    $fio->{rxSize} = $attr->{size};
    delete($fio->{rxInFd});
    delete($fio->{rxOutFd});
    delete($fio->{rxDigest});
    delete($fio->{rxInData});
    return $fio->fileDeltaRxDone();
}

#
# Start receive of file deltas for a particular file.
#
sub fileDeltaRxStart
{
    my($fio, $f, $cnt, $size, $remainder) = @_;

    $fio->{rxFile}      = $f;           # remote file attributes
    $fio->{rxLocalAttr} = $fio->attribGet($f); # local file attributes
    $fio->{rxBlkCnt}    = $cnt;         # how many blocks we will receive
    $fio->{rxBlkSize}   = $size;        # block size
    $fio->{rxRemainder} = $remainder;   # size of the last block
    $fio->{rxMatchBlk}  = 0;            # current start of match
    $fio->{rxMatchNext} = 0;            # current next block of match
    $fio->{rxSize}      = 0;            # size of received file
    my $rxSize = $cnt > 0 ? ($cnt - 1) * $size + $remainder : 0;
    if ( $fio->{rxFile}{size} != $rxSize ) {
        $fio->{rxMatchBlk} = undef;     # size different, so no file match
        $fio->log("$fio->{rxFile}{name}: size doesn't match"
                  . " ($fio->{rxFile}{size} vs $rxSize)")
                        if ( $fio->{logLevel} >= 5 );
    }
    #
    # If compression was off and now on, or on and now off, then
    # don't do an exact match.
    #
    if ( defined($fio->{rxLocalAttr})
	    && !$fio->{rxLocalAttr}{compress} != !$fio->{xfer}{compress} ) {
        $fio->{rxMatchBlk} = undef;     # compression changed, so no file match
        $fio->log("$fio->{rxFile}{name}: compression changed, so no match"
              . " ($fio->{rxLocalAttr}{compress} vs $fio->{xfer}{compress})")
                    if ( $fio->{logLevel} >= 4 );
    }
    #
    # If the local file is a hardlink then no match
    #
    if ( defined($fio->{rxLocalAttr})
	    && $fio->{rxLocalAttr}{type} == BPC_FTYPE_HARDLINK ) {
        $fio->{rxMatchBlk} = undef;
        $fio->log("$fio->{rxFile}{name}: no match on hardlinks")
                                    if ( $fio->{logLevel} >= 4 );
        my $fCopy;
        # need to copy since hardlink attribGet overwrites the name
        %{$fCopy} = %$f;
        $fio->{rxHLinkAttr} = $fio->attribGet($fCopy, 1); # hardlink attributes
    } else {
        delete($fio->{rxHLinkAttr});
    }
    delete($fio->{rxInFd});
    delete($fio->{rxOutFd});
    delete($fio->{rxDigest});
    delete($fio->{rxInData});
}

#
# Process the next file delta for the current file.  Returns 0 if ok,
# -1 if not.  Must be called with either a block number, $blk, or new data,
# $newData, (not both) defined.
#
sub fileDeltaRxNext
{
    my($fio, $blk, $newData) = @_;

    if ( defined($blk) ) {
        if ( defined($fio->{rxMatchBlk}) && $fio->{rxMatchNext} == $blk ) {
            #
            # got the next block in order; just keep track.
            #
            $fio->{rxMatchNext}++;
            return;
        }
    }
    my $newDataLen = length($newData);
    $fio->log("$fio->{rxFile}{name}: blk=$blk, newData=$newDataLen, rxMatchBlk=$fio->{rxMatchBlk}, rxMatchNext=$fio->{rxMatchNext}")
		    if ( $fio->{logLevel} >= 8 );
    if ( !defined($fio->{rxOutFd}) ) {
	#
	# maybe the file has no changes
	#
	if ( $fio->{rxMatchNext} == $fio->{rxBlkCnt}
		&& !defined($blk) && !defined($newData) ) {
	    #$fio->log("$fio->{rxFile}{name}: file is unchanged");
	    #		    if ( $fio->{logLevel} >= 8 );
	    return;
	}

        #
        # need to open an output file where we will build the
        # new version.
        #
        $fio->{rxFile}{name} =~ /(.*)/s;
	my $rxOutFileRel = "$fio->{shareM}/" . $fio->{bpc}->fileNameMangle($1);
        my $rxOutFile    = $fio->{outDir} . $rxOutFileRel;
        $fio->{rxOutFd}  = BackupPC::PoolWrite->new($fio->{bpc},
					   $rxOutFile, $fio->{rxFile}{size},
                                           $fio->{xfer}{compress});
        $fio->log("$fio->{rxFile}{name}: opening output file $rxOutFile")
                        if ( $fio->{logLevel} >= 9 );
        $fio->{rxOutFile} = $rxOutFile;
        $fio->{rxOutFileRel} = $rxOutFileRel;
        $fio->{rxDigest} = File::RsyncP::Digest->new();
        $fio->{rxDigest}->protocol($fio->{protocol_version});
        $fio->{rxDigest}->add(pack("V", $fio->{checksumSeed}));
    }
    if ( defined($fio->{rxMatchBlk})
                && $fio->{rxMatchBlk} != $fio->{rxMatchNext} ) {
        #
        # Need to copy the sequence of blocks that matched.  If the file
        # is compressed we need to make a copy of the uncompressed file,
        # since the compressed file is not seekable.  Future optimizations
        # could include only creating an uncompressed copy if the matching
        # blocks were not monotonic, and to only do this if there are
        # matching blocks (eg, maybe the entire file is new).
        #
        my $attr = $fio->{rxLocalAttr};
	my $fh;
        if ( !defined($fio->{rxInFd}) && !defined($fio->{rxInData}) ) {
            my $inPath = $attr->{fullPath};
            $inPath = $fio->{rxHLinkAttr}{fullPath}
                            if ( defined($fio->{rxHLinkAttr}) );
            if ( $attr->{compress} ) {
                if ( !defined($fh = BackupPC::FileZIO->open(
                                                   $inPath,
                                                   0,
                                                   $attr->{compress})) ) {
                    $fio->log("Can't open $inPath");
		    $fio->{stats}{errorCnt}++;
                    return -1;
                }
                if ( $attr->{size} < 16 * 1024 * 1024 ) {
                    #
                    # Cache the entire old file if it is less than 16MB
                    #
                    my $data;
                    $fio->{rxInData} = "";
                    while ( $fh->read(\$data, 16 * 1024 * 1024) > 0 ) {
                        $fio->{rxInData} .= $data;
                    }
		    $fio->log("$attr->{fullPath}: cached all $attr->{size}"
			    . " bytes")
				    if ( $fio->{logLevel} >= 9 );
                } else {
                    #
                    # Create and write a temporary output file
                    #
                    unlink("$fio->{outDirSh}RStmp")
                                    if  ( -f "$fio->{outDirSh}RStmp" );
                    if ( open(F, "+>", "$fio->{outDirSh}RStmp") ) {
                        my $data;
			my $byteCnt = 0;
			binmode(F);
                        while ( $fh->read(\$data, 1024 * 1024) > 0 ) {
                            if ( syswrite(F, $data) != length($data) ) {
                                $fio->log(sprintf("Can't write len=%d to %s",
				      length($data) , "$fio->{outDirSh}RStmp"));
                                $fh->close;
				$fio->{stats}{errorCnt}++;
                                return -1;
                            }
			    $byteCnt += length($data);
                        }
                        $fio->{rxInFd} = *F;
                        $fio->{rxInName} = "$fio->{outDirSh}RStmp";
                        sysseek($fio->{rxInFd}, 0, 0);
			$fio->log("$attr->{fullPath}: copied $byteCnt,"
				. "$attr->{size} bytes to $fio->{rxInName}")
					if ( $fio->{logLevel} >= 9 );
                    } else {
                        $fio->log("Unable to open $fio->{outDirSh}RStmp");
                        $fh->close;
			$fio->{stats}{errorCnt}++;
                        return -1;
                    }
                }
                $fh->close;
            } else {
                if ( open(F, "<", $inPath) ) {
		    binmode(F);
                    $fio->{rxInFd} = *F;
                    $fio->{rxInName} = $attr->{fullPath};
                } else {
                    $fio->log("Unable to open $inPath");
		    $fio->{stats}{errorCnt}++;
                    return -1;
                }
            }
        }
	my $lastBlk = $fio->{rxMatchNext} - 1;
        $fio->log("$fio->{rxFile}{name}: writing blocks $fio->{rxMatchBlk}.."
                  . "$lastBlk")
                        if ( $fio->{logLevel} >= 9 );
        my $seekPosn = $fio->{rxMatchBlk} * $fio->{rxBlkSize};
        if ( defined($fio->{rxInFd})
			&& !sysseek($fio->{rxInFd}, $seekPosn, 0) ) {
            $fio->log("Unable to seek $fio->{rxInName} to $seekPosn");
	    $fio->{stats}{errorCnt}++;
            return -1;
        }
        my $cnt = $fio->{rxMatchNext} - $fio->{rxMatchBlk};
        my($thisCnt, $len, $data);
        for ( my $i = 0 ; $i < $cnt ; $i += $thisCnt ) {
            $thisCnt = $cnt - $i;
            $thisCnt = 512 if ( $thisCnt > 512 );
            if ( $fio->{rxMatchBlk} + $i + $thisCnt == $fio->{rxBlkCnt} ) {
                $len = ($thisCnt - 1) * $fio->{rxBlkSize} + $fio->{rxRemainder};
            } else {
                $len = $thisCnt * $fio->{rxBlkSize};
            }
            if ( defined($fio->{rxInData}) ) {
                $data = substr($fio->{rxInData}, $seekPosn, $len);
		$seekPosn += $len;
            } else {
		my $got = sysread($fio->{rxInFd}, $data, $len);
                if ( $got != $len ) {
		    my $inFileSize = -s $fio->{rxInName};
                    $fio->log("Unable to read $len bytes from $fio->{rxInName}"
                            . " got=$got, seekPosn=$seekPosn"
                            . " ($i,$thisCnt,$fio->{rxBlkCnt},$inFileSize"
			    . ",$attr->{size})");
		    $fio->{stats}{errorCnt}++;
                    return -1;
                }
		$seekPosn += $len;
            }
            $fio->{rxOutFd}->write(\$data);
            $fio->{rxDigest}->add($data);
	    $fio->{rxSize} += length($data);
        }
        $fio->{rxMatchBlk} = undef;
    }
    if ( defined($blk) ) {
        #
        # Remember the new block number
        #
        $fio->{rxMatchBlk}  = $blk;
        $fio->{rxMatchNext} = $blk + 1;
    }
    if ( defined($newData) ) {
        #
        # Write the new chunk
        #
        my $len = length($newData);
        $fio->log("$fio->{rxFile}{name}: writing $len bytes new data")
                        if ( $fio->{logLevel} >= 9 );
        $fio->{rxOutFd}->write(\$newData);
        $fio->{rxDigest}->add($newData);
	$fio->{rxSize} += length($newData);