calibrationcurve.cpp

粗糙集应用软件

	}

	// Sort vectors by mean targets per group.
	permuter.Permute(targets_mean_,   permutation);

	MathKit::Permute(targets_mean_,   permutation);
	MathKit::Permute(outputs_mean_,   permutation);
	MathKit::Permute(indices_summed_, permutation);
	MathKit::Permute(group_sizes_,    permutation);

	// Set decision class.
	SetDecisionClass(decision_class);

	return true;

}

//-------------------------------------------------------------------
// Method........: Save
// Author........: Aleksander 豩rn
// Date..........:
// Description...:
// Comments......:
// Revisions.....:
//===================================================================

bool
CalibrationCurve::Save(ofstream &stream, Id id, const String &parameters) const {

	float mean_0s, mean_1s, variance_0s, variance_1s, regression_slope, regression_intercept;

	// Compute stuff.
	float mean_target        = ComputeMeanTarget();
	float mean_output        = ComputeMeanOutput();
	float brier_score        = ComputeBrierScore();
	float covariance_bias    = ComputeBias();
	float covariance_slope   = ComputeSlope(&mean_0s, &mean_1s);
	float covariance_scatter = ComputeScatter(&variance_0s, &variance_1s);

	if (!ComputeLinearRegression(&regression_slope, &regression_intercept)) {
		regression_slope     = Undefined::Float();
		regression_intercept = Undefined::Float();
	}

	// Determine output strings.
	String brier_string                = (brier_score          == Undefined::Float()) ? Undefined::String() : String::Format(brier_score);
	String covariance_bias_string      = (covariance_bias      == Undefined::Float()) ? Undefined::String() : String::Format(covariance_bias);
	String covariance_slope_string     = (covariance_slope     == Undefined::Float()) ? Undefined::String() : String::Format(covariance_slope);
	String covariance_scatter_string   = (covariance_scatter   == Undefined::Float()) ? Undefined::String() : String::Format(covariance_scatter);
	String regression_slope_string     = (regression_slope     == Undefined::Float()) ? Undefined::String() : String::Format(regression_slope);
	String regression_intercept_string = (regression_intercept == Undefined::Float()) ? Undefined::String() : String::Format(regression_intercept);

	// Notify user?
	if (Configuration::Kernel::IsVerbose()) {
		Message message;
		message.Notify("Brier score = " + brier_string);
		message.Notify("Bias = " + covariance_bias_string);
		message.Notify("Slope = " + covariance_slope_string);
		message.Notify("Scatter = " + covariance_scatter_string);
		message.Notify("Regression slope = " + regression_slope_string);
		message.Notify("Regression intercept = " + regression_intercept_string);
	}

	// Save header.
	stream << "% Output from ROSETTA, " + SystemKit::GetUser() + " " + SystemKit::GetTimestamp() << endl;
	stream << "%" << endl;
	stream << "% " + IdHolder::GetClassname(id) << endl;
	stream << "% {" + parameters + "}" << endl;
	stream << "%" << endl;
	stream << "% Format:" << endl;
	stream << "%    <Mean target/actual outcome per group> <Mean model output per group> <Sum of object indices per group> <Group size>" << endl;
	stream << "%" << endl;
	stream << "% Targets and outputs are binarized with respect to the selected decision class." << endl;
	stream << "%" << endl;
	stream << "% Brier score                      = " << brier_string << endl;
	stream << "% Mean outcome                     = " << String((mean_target == Undefined::Float()) ? Undefined::String() : String::Format(mean_target)) << endl;
	stream << "% Mean prediction                  = " << String((mean_output == Undefined::Float()) ? Undefined::String() : String::Format(mean_output)) << endl;
	stream << "% Mean of model outputs for 0s     = " << String((mean_0s     == Undefined::Float()) ? Undefined::String() : String::Format(mean_0s))     << endl;
	stream << "% Mean of model outputs for 1s     = " << String((mean_1s     == Undefined::Float()) ? Undefined::String() : String::Format(mean_1s))     << endl;
	stream << "% Variance of model outputs for 0s = " << String((variance_0s == Undefined::Float()) ? Undefined::String() : String::Format(variance_0s)) << endl;
	stream << "% Variance of model outputs for 1s = " << String((variance_1s == Undefined::Float()) ? Undefined::String() : String::Format(variance_1s)) << endl;
	stream << "% Bias                             = " << covariance_bias_string << endl;
	stream << "% Slope                            = " << covariance_slope_string << endl;
	stream << "% Scatter                          = " << covariance_scatter_string << endl;
	stream << "%" << endl;
	stream << "% For a reference to bias, slope and scatter, see:" << endl;
	stream << "%" << endl;
  stream << "% H. R. Arkes et al. (1995), \"The Covariance Decomposition of the Probability Score" << endl;
	stream << "% and Its Use in Evaluating Prognostic Estimates\", Medical Decision Making, 15, pp. 120-131." << endl;
	stream << "%" << endl;
	stream << "% Linear regression equation:" << endl;
	stream << "%    <Mean model output per group> = " << regression_slope_string << " * <Mean target/actual outcome per group> + " << regression_intercept_string << endl;
	stream << endl;

	int i;

	// Save data.
	for (i = 0; i < GetNoGroups(); i++) {
		stream << GetMeanTarget(i) << ' ';
		stream << GetMeanOutput(i) << ' ';
		stream << GetIndexSum(i)  << ' ';
		stream << GetGroupSize(i)  << endl;
	}

	return true;
}

//-------------------------------------------------------------------
// Method........: Save
// Author........: Aleksander 豩rn
// Date..........:
// Description...:
// Comments......:
// Revisions.....:
//===================================================================

bool
CalibrationCurve::Save(const String &filename, Id id, const String &parameters) const {

	ofstream stream;

	if (!IOKit::Open(stream, filename))
		return false;

	return Save(stream, id, parameters);

}

//-------------------------------------------------------------------
// Method........: ComputeBrierScore
// Author........: Aleksander 豩rn
// Date..........:
// Description...:
// Comments......: The Brier score is simply the average squared
//                 difference between target and output.
// Revisions.....:
//===================================================================

float
CalibrationCurve::ComputeBrierScore() const {

	int i, no_pairs = GetNoPairs();

	if (no_pairs == 0)
		return 0.0;

	double sum = 0.0;

	for (i = 0; i < no_pairs; i++) {
		double difference = GetTarget(i) - GetOutput(i);
		sum += difference * difference;
	}

	return static_cast(float, sum / no_pairs);

}

//-------------------------------------------------------------------
// Method........: ComputeMeanTarget/ComputeMeanOutput
// Author........: Aleksander 豩rn
// Date..........:
// Description...:
// Comments......:
// Revisions.....:
//===================================================================

float
CalibrationCurve::ComputeMeanTarget() const {
	return MathKit::Mean(GetTargets());
}

float
CalibrationCurve::ComputeMeanOutput() const {
	return MathKit::Mean(GetOutputs());
}

//-------------------------------------------------------------------
// Method........: ComputeBias
// Author........: Aleksander 豩rn
// Date..........:
// Description...: Bias is the difference between the average model
//                 prediction and the average outcome.
//
// Comments......: Arkes et al. (1995), "The Covariance Decomposition
//                 of the Probability Score and Its Use in Evaluating
//                 Prognostic Estimates", Medical Decision Making,
//                 15, pp. 120-131.
//
//                 Lower absolute values of bias are better.
// Revisions.....:
//===================================================================

float
CalibrationCurve::ComputeBias() const {

	if (GetNoPairs() == 0)
		return 0.0;

	float mean_output = ComputeMeanOutput();
	float mean_target = ComputeMeanTarget();

	return mean_output - mean_target;

}

//-------------------------------------------------------------------
// Method........: ComputeSlope
// Author........: Aleksander 豩rn
// Date..........:
// Description...: Slope is the average model prediction assigned
//                 to those with outcome 1 (the decision class),
//                 minus the average model prediction assigned to those
//                 with outcome 0 (not the decision class).
//
//                 The average model predictions for 0s and 1s are
//                 returned in-place, if specified.
//
// Comments......: See paper by Arkes et al.
//
//                 Higher slope is better.
// Revisions.....:
//===================================================================

float
CalibrationCurve::ComputeSlope(float *mean_0s, float *mean_1s) const {

	int i, no_pairs = GetNoPairs();

	double f0 = 0.0;
	double f1 = 0.0;

	int n0 = 0;
	int n1 = 0;

	for (i = 0; i < no_pairs; i++) {
		float output = GetOutput(i);
		if (GetTarget(i) == 1) {
			f1 += output;
			n1++;
		}
		else {
			f0 += output;
			n0++;
		}
	}

	if (n0 == 0 || n1 == 0) {
		if (mean_0s != NULL)
			*mean_0s = Undefined::Float();
		if (mean_1s != NULL)
			*mean_1s = Undefined::Float();
		return Undefined::Float();
	}

	float m_0s = static_cast(float, f0 / n0);
	float m_1s = static_cast(float, f1 / n1);

	if (mean_0s != NULL)
		*mean_0s = m_0s;
	if (mean_1s != NULL)
		*mean_1s = m_1s;

	return m_1s - m_0s;

}

//-------------------------------------------------------------------
// Method........: ComputeScatter
// Author........: Aleksander 豩rn
// Date..........:
// Description...: Scatter is an index of the "noisiness" of the
//                 model predictions. Scatter is a pooled variance
//                 derived from the distribution of model predictions
//                 assigned to those with outcome 1 and the distribution
//                 of model predictions assigned to those with outcome 0.
//                 The variance of each distribution is calculated
//                 separately before they are pooled. This means that
//                 scatter is unrelated to whether or the target event
//                 occurred.
//
//                 The variances of the model predictions for 0s and 1s
//                 are returned in-place, if specified.
//
// Comments......: See paper by Arkes et al.
// Revisions.....:
//===================================================================

float
CalibrationCurve::ComputeScatter(float *variance_0s, float *variance_1s) const {

	int i, no_pairs = GetNoPairs();

	Vector(float) f0;
	Vector(float) f1;

	f0.reserve(no_pairs);
	f1.reserve(no_pairs);

	int n0 = 0;
	int n1 = 0;

	for (i = 0; i < no_pairs; i++) {
		float output = GetOutput(i);
		if (GetTarget(i) == 1) {
			f1.push_back(output);
			n1++;
		}
		else {
			f0.push_back(output);
			n0++;
		}
	}

	float v_0s = MathKit::Variance(f0);
	float v_1s = MathKit::Variance(f1);

	if (variance_0s != NULL)
		*variance_0s = v_0s;
	if (variance_1s != NULL)
		*variance_1s = v_1s;

	if (v_0s == Undefined::Float() || v_1s == Undefined::Float())
		return 0.0;

	if (n0 + n1 == 0)
		return 0.0;

	return ((n0 * v_0s) + (n1 * v_1s)) / (n0 + n1);

}

//-------------------------------------------------------------------
// Method........: ComputeLinearRegression
// Author........: Aleksander 豩rn
// Date..........:
// Description...: Given the points on the calibration curve,
//                 computes the slope and intercept of the
//                 corresponding linear regression equation.
// Comments......:
// Revisions.....:
//===================================================================

bool
CalibrationCurve::ComputeLinearRegression(float *slope, float *intercept) const {

	float s, i;

	if (!MathKit::LinearRegression(targets_mean_, outputs_mean_, s, i))
		return false;

	if (slope != NULL)
		*slope = s;
	if (intercept != NULL)
		*intercept = i;

	return true;

}