read_ap_txt.m

上载文件为Matlab环境下的高斯以马尔科夫模型例程

function [data, textlines, Nalphabet] = read_ap_txt(filename, max_lines)
% READ_AP_TXT Read AP news file and convert to numeric format
% [data, textlines, Nalphabet] = read_ap_txt(filename, max_lines)
%
% If max_lines is omitted, the whole file is read.
% data{m} is an integer representation of the m'th sentence.
% textlines{m} is the original text of line m
% Nalphabet is the number of unique symbols used in the encoding.
 
if nargin < 2, max_num_lines = inf; end

% textlines{m} is the m'th line of text
textlines = textread(filename, '%s','delimiter','\n','whitespace','');
textlines = lower(textlines); % convert to lower case
nlines = length(textlines);

% convert each character to a number
% asciimap(a) = i means ascii code a will get mapped to integer i
% inverse_asciimap(i) = a means integer i is letter a
asciimap = zeros(1,128);
punctuation = [32:47 58:64 91:96 123:127];
digits = 48:57;
uppercase = 65:90;
lowercase = 97:122;

i = 1;
for a=lowercase
  asciimap(a) = i;
  inverse_asciimap(i) = char(a);
  i = i + 1;
end

if 0
  for a=digits
    asciimap(a) = i;
    inverse_asciimap(i) = char(a);
    i = i + 1;
  end
else
  % map all numbers to a single number (which will print as 0)
  for a=digits
    asciimap(a) = i;
    inverse_asciimap(i) = '0';
  end
  i = i + 1;
end

% map all punctuation to a single number (which will print as *)
for a=punctuation
  asciimap(a) = i;
  inverse_asciimap(i) = '*';
end

Nalphabet = i;
%assert(Nalphabet == length(lowercase) + length(digits) + 1) % 37
assert(Nalphabet == length(lowercase) + 1 + 1) % 28

nlines = min(nlines, max_lines);

data = cell(1, nlines);
for l=1:nlines
  data{l} = asciimap(double(textlines{l}(1:end-1))); % strip off final \n 
  if any(data{l} < 1) | any(data{l} > Nalphabet)
    error(['problem with line ' num2str(l) ': ' textlines{l}]);
  end
end