chardistribution.cpp

判断一串字符是属于什么字符集的程序

/*
	libcharguess	-	Guess the encoding/charset of a string
    Copyright (C) 2003  Stephane Corbe <[e-mail]>
	Based on Mozilla sources

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2.1 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "charDistribution.h"

#include "JISFreq.tab"
#include "Big5Freq.tab"
#include "EUCKRFreq.tab"
#include "EUCTWFreq.tab"
#include "GB2312Freq.tab"

#define SURE_YES 0.99f
#define SURE_NO  0.01f

//return confidence base on received data
float CharDistributionAnalysis::GetConfidence()
{ 
  //if we didn't receive any character in our consideration range, return negative answer
  if (mTotalChars <= 0)
    return SURE_NO;

  if (mTotalChars != mFreqChars) {
    float r = mFreqChars / ((mTotalChars - mFreqChars) * mTypicalDistributionRatio);

    if (r < SURE_YES)
      return r;
  }
  //normalize confidence, (we don't want to be 100% sure)
  return SURE_YES;
}

EUCTWDistributionAnalysis::EUCTWDistributionAnalysis()
{
  mCharToFreqOrder = EUCTWCharToFreqOrder;
  mTableSize = EUCTW_TABLE_SIZE;
  mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO;
};

EUCKRDistributionAnalysis::EUCKRDistributionAnalysis()
{
  mCharToFreqOrder = EUCKRCharToFreqOrder;
  mTableSize = EUCKR_TABLE_SIZE;
  mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO;
};

GB2312DistributionAnalysis::GB2312DistributionAnalysis()
{
  mCharToFreqOrder = GB2312CharToFreqOrder;
  mTableSize = GB2312_TABLE_SIZE;
  mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO;
};

Big5DistributionAnalysis::Big5DistributionAnalysis()
{
  mCharToFreqOrder = Big5CharToFreqOrder;
  mTableSize = BIG5_TABLE_SIZE;
  mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO;
};

SJISDistributionAnalysis::SJISDistributionAnalysis()
{
  mCharToFreqOrder = JISCharToFreqOrder;
  mTableSize = JIS_TABLE_SIZE;
  mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO;
};

EUCJPDistributionAnalysis::EUCJPDistributionAnalysis()
{
  mCharToFreqOrder = JISCharToFreqOrder;
  mTableSize = JIS_TABLE_SIZE;
  mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO;
};