normal.pm

harvest是一个下载html网页得机器人

package normal;

sub normalize_uri {

    my $uri = shift;

    $uri =~ s/ /%20/g;       # spaces
    $uri =~ s/\/+$/\//g;     # trailing //

    my ($proto, $host, $path, $file) = $uri =~ /(\S+):\/\/(\S+?)\/(\S*)\/(\S*)/;

    return "$uri" if ($path eq "");

    do {
	$path = normalize_path ($path);
    } while ($path =~ /\.\./);

    $mypath = "$host/$path/$file";
    $mypath =~ s#/+#/#g;

    return "$proto:\/\/$mypath";
}

sub normalize_path {

    my $path = shift;

    $path =~ s/\/\.\//\//g;    # /./
    $path =~ s/\/+/\//g;       # //
    $path =~ s/^\///g;         # leading /
    $path =~ s/^\.\.\///g;     # leading ../

    if ($path =~ /\.\./) {
	@full_path = split /\//, $path;

	$pos = 0;
	$path = "";
	foreach $i ($pos .. $#full_path) {
	    if ($full_path[$i] eq "..") {
		$pos = $i;
		last;
	    }
	}

	foreach $i (0 .. $pos - 2) {
	    $path  .= "/$full_path[$i]";
	}

	foreach $i ($pos + 1 .. $#full_path) {
	    $path .= "/$full_path[$i]";
	}

    }

    return $path;
}

1