quickclustering.h

MS-Clustering is designed to rapidly cluster large MS/MS datasets. The program merges similar spectr

void set_adjusted_inten(QCPeak *peaks, int num_peaks);

// Sets the adusted intensity of the peaks that are in top_ranked_idxs
// the spectrum is reduced only to these peaks, so the intensity given to each peak
// is the ration I/I_total where I_total is for all the peaks in the top_ranked_idxs
void set_top_ranked_peak_inten(QCPeak *peaks, int num_peaks, const vector<int>& top_ranked_idxs, 
							   vector<float>& top_ranked_peak_inten);


float calc_sum_adjusted_inten_squared(const QCPeak *peaks, int num_peaks);

float calc_sum_adjusted_inten_squared(const QCPeak *peaks, int num_peaks, 
									  const vector<int>& top_ranked_peak_idxs);




float calc_selected_dot_prod(mass_t tolerance, 
							 const QCPeak *pa, int na, const vector<int>& peak_idxs_a,
	  					     const QCPeak* pb, int nb, const vector<int>& peak_idxs_b,
							 bool verbose = false);


void collect_dot_product_stats(char *list_file);


void dot_prod_exp();



// Outputs clusters 
class QCOutputter {
public:
	QCOutputter() : init_flag(false), name("clusters"), 
		dir("."), batch_idx(0), spectra_counter(0), total_spectra_counter(0), file_counter(0) {};

	~QCOutputter();
	
	void init(string _name , string _dir, int _batch_idx=-1,
			  mass_t min_m_over_z=POS_INF, mass_t max_m_over_z=NEG_INF,
			  float min_similarity=-1, int min_cluster_size=0);

	void output_basic_spectrum_to_mgf(BasicSpectrum &bs, Config *config);

	void output_cluster_spectrum(ClusterSpectrum& cs);

	void output_cluster_spectrum_as_single_pkl(ClusterSpectrum& cs);

	void output_cluster_anns(ClusterSpectrum& cs);


private:
	bool   init_flag;
	
	string name;
	string dir;
	string mgf_name;
	string cluster_file_name;
	string batch_str;

	int    batch_idx;
	int    spectra_counter;
	int    total_spectra_counter;
	int	   file_counter;
	
	fstream mgf_stream;           // mgf containing cluster consensus spectra
	fstream cluster_file_stream;  // file listing foreach cluster what single spectra belong to it 
	fstream summary_stream;               // for each consensus spectrum lists how many single spectra belong to it
	fstream file_list_stream;             // a list of all the mgf files created
	fstream anns_stream;                  // stream for annotations 
};





class DAT_FileBuff {
	friend class DAT_Converter;
public:

	DAT_FileBuff()  : buff(NULL), max_pos(NULL), pos(NULL), counter(0),
					  ind_first_write(1), ind_was_initialized(0) {};
	~DAT_FileBuff();

	void init(string& _path, int buff_size); // initializes ans allocates buffer memory

	// copies the files to the DAT file
	void add_spec_to_DAT_file(mass_t m_over_z, int charge, int mzxml_file_idx,
							  int scan_number, float retention_time, 
							  float precursor_intensity, int num_peaks, char *peak_buff);

	void flush_buff(); // writes buff to file
private:

	string path;
	char *buff;
	char *max_pos;
	char *pos;

	int  counter;
	int  ind_first_write;      // is this the first write (if not, append)
	int  ind_was_initialized;  
};


class DAT_Converter {
public:

	DAT_Converter() : max_m_over_z((mass_t)2000.0), mass_increment((mass_t)DAT_FILE_INCREMENT),
		dat_buff_size(DAT_BUFF_SIZE), max_dat_file_idx(-1), ind_was_initialized(0),
		batch(0) {};

	~DAT_Converter()
	{
		int d;
		for (d=0; d<dat_buffs.size(); d++)
			if (dat_buffs[d].ind_was_initialized && dat_buffs[d].pos > dat_buffs[d].buff)
				dat_buffs[d].flush_buff();
	}

	void init_DAT_Converter(mass_t _max_m_over_z, mass_t _mass_increment, 
			  int dat_buff_size);

	void convert_files_to_DAT_on_the_fly(Config* config, char *file_list, 
							char * _out_dir, char * _name, int _batch, 
							mass_t min_m_over_z, mass_t max_m_over_z, int file_start_idx,
							bool ind_is_pkl_dir);



	int convert_PKL_dir_to_DAT(Config* config, char *file_list, int file_start_idx,
							mass_t min_m_over_z, mass_t max_m_over_z);


	int convert_single_non_MZXML_file_to_DAT(Config* config, string file, 
							mass_t min_m_over_z, mass_t max_m_over_z, int file_idx);

	int parse_annotated_spectra_from_single_MZXML(
								Config *config, 
								string& mzxml_name, 
								int file_idx,
								map<mzXML_annotation,int>& ann_map);

	void create_dat_files_for_anns(Config *config, char *mzXML_list, char *anns_file,
								char *_out_dir, char *_name);



	// creates a file with the list of DAT files
	void create_list_file() const;

private:
	string out_dir;
	string name;

	mass_t max_m_over_z;
	mass_t mass_increment;
	int    dat_buff_size;
	int    max_dat_file_idx;
	int    batch;

	int    ind_was_initialized;

	vector<DAT_FileBuff> dat_buffs;

	int parse_single_MZXML_file(Config *config, string& mzxml_name, int file_idx,
								mass_t min_m_over_z, mass_t max_m_over_z);
};

/////////////////////////////////////////////////////////////////////////////
// For mzXML parsing
struct MassInten {
	float mass, intensity;
};


int join_and_filter_peak_list(Config *config, 
							  mass_t m_over_z, 
							  float *org_peaks, 
							  int num_org_peaks, 
							  float *new_peaks);
/////////////////////////////////////////////////////////////////////////////




/***************************************************************************
	This function creates clusters from a list of files containing spectra
	(possibly different file types).
	The cluster spectra are outputted as mgf files in the output dir (x spectra
	per file). In addition, for each cluster file there is a map file that holds
	the indices (position in list, and idx in file) of the original spectra
	that are part of the cluster.
****************************************************************************/
void cluster_full_dataset(Config *config,
							  char *list_file,
							  const string& out_dir,
							  const string& clust_name,
							  int batch_idx,
							  int max_spec_per_slice = 20000,
							  mass_t min_m_over_z = 0,
							  mass_t max_m_over_z = 10000,
							  float  min_similarity = 0.8, 
							  int min_cluster_size = 2,
							  int max_cluster_size = 1000,
							  bool verbose = false,
							  int  max_small_cluster_size = -1,
							  int  k_value = 20,
							  void *pmcsqs = NULL,
							  bool ind_pkl_mode = false,
							  float filter_prob = 0.075,
							  int max_mzxml_idx = -1,
							  char *good_anns_file = NULL);


/**************************************************************************
Given a query spectrum finds the x number of spectra that have the
highest similarity to the query
***************************************************************************/
void find_spectra_with_max_similarity_to_query(
							  char *model_name,
							  char *dat_list,
							  char *query_spectrum,
							  int num_top_spectra);


int cluster_spec_in_file_set(Config *config, const FileManager& fm, FileSet& cluster_fs,
							  mass_t tolerance, 
							  QCPeak *basic_peaks,
							  vector<ClusterSpectrum>& clusters, 
							  float min_similarity,
							  int   max_small_cluster_size,
							  int	max_cluster_size,
							  int   num_top_peaks_per_1000_da,
							  bool verbose,
							  void *pmcqsqs = NULL,
							  float filter_prob = 0.075,
							  map<mzXML_annotation,int> *ann_map_ptr = NULL);



int add_additional_spectra_to_existing_clusters(Config *config, const FileManager& fm, 
							  FileSet& additional_fs, mass_t tolerance, QCPeak *basic_peaks, 
							  vector<ClusterSpectrum>& clusters, 
							  float min_similarity,
							  int max_cluster_size = 100000,
							  void *pmcqsqs = NULL,
							  float filter_prob = 0.075,
							  map<mzXML_annotation,int> *ann_map_ptr = NULL,
							  bool verbose = false);



// reading mzXML, mgf files,  annotations




bool read_mgf_file_into_basic_spectra(Config *config, char *mgf_file, QCPeak *basic_peaks,
									  vector<BasicSpectrum>& basic_spectra);

void ann_mgf_and_create_mgf(Config *config, char *annotations_file, char *mgf_list_file,
							  char *out_dir_name, char *file_prefix, bool output_only_ann_spectra = false);


void read_mzXML_annotations(char *mzXML_list,char *ann_file, vector< vector<int> >& annotation_idxs, 
							vector<mzXML_annotation>& annotations, int max_ann_size = 50000);

void read_mzXML_annotations_to_map(char *ann_file, map<mzXML_annotation,int>& ann_map);

void read_mzXML_annotations_limited(char *mzXML_list, 
							char *ann_file, 
							vector< vector<int> >& annotation_idxs, 
							vector<mzXML_annotation>& annotations);

void extract_spectra_stats_from_mzXML(Config *config, char *annotations_file, 
									  char *mzXML_list_file, char *stat_file);

void read_annotated_dataset_into_clusters(Config *config, FileManager& fm,
				const vector<SingleSpectrumFile *>& all_ssf, char *ann_mgf, 
				int num_specs_per_cluster, int max_num_clusters, vector<ClusterSpectrum>& clusters);

void benchmark_similarity_measures(Config *config, char *ann_mgf, 
								   int num_specs_per_cluster);

void benchmark_inter_similarity_vs_outer_similarity(Config *config, char *ann_mgf, 
								   int num_specs_per_cluster);

void benchmark_similarity_to_consensus(Config *config, char *ann_mgf, 
								   int num_specs_per_cluster);

void benchmark_signal_to_noise(void *model, char *ann_mgf, 
								   int num_specs_per_cluster);

void benchmark_signal(void *model, char *ann_mgf, 
								   int num_specs_per_cluster);
  


void print_specs_from_mgf(Config *config,char *mgf_name);



void qc_exp();

void qc_ann_exp(char *name, bool verbose = false);

void benchmark_large_clusters(void *model_ptr, char *ann_mgf, 
								   int num_specs_per_cluster);

void create_spectra_mgf_file(void *model_ptr, char *ann_mgf, 
								   int num_specs_per_cluster);

void create_file_with_rt_scores(void *model_ptr, char *ann_mgf, 
								   int num_specs_per_cluster);

// checks that the anntoated spectra have a correct m_over_z
void print_specs(Config *config, char *list_name);

void read_ms2(Config *config, char *ms2_list);

void ann_mgf_and_create_mgf_with_sim_masses(Config *config, 
											char *annotations_file, 
											char *org_mgf_list_file,
											char *good_peptide_list,
											char *out_dir_name, 
											char *file_prefix);

void make_specified_benchmark_clustering_dataset(
				Config *config, 
				char *ann_mgf_list,
				mass_t min_m_over_z, 
				mass_t max_m_over_z,
				char *out_dir_name, 
				char *file_prefix,
				int num_clusters,
				int cluster_size,
				int number_non_assigned_per_assigned);

void make_benchmark_clustering_dataset(Config *config, char *ann_mgf_list,
				mass_t min_m_over_z, mass_t max_m_over_z,bool only_confident_anns,
				char *out_dir_name, char *file_prefix);

void benchmark_clustering_performance(Config *config,
							  char *list_file,
							  int  k_value);

void benchmark_k_value(Config *config, char *list_file);

void benchmark_heuristic_filtering(Config *config, char *dat_list);

void benchmark_top7_and_sim_thresh(Config *config, char *dat_list, char *ann_file);

void benchmark_retention_thresh(Config *config, char *dat_list, char *ann_file);

void find_pair_similarities(Config *config, char *mgf_file, char *pair_file);

void extractAnnoatedScansFromFiles(Config *config, char *file_list, char *anns, 
								   char *output_file, bool extract_no_process);

void extract_annotate_scans_from_dat(Config *config, char *dat_list, char *anns_file, 
									 char *out_mgf_file);

void create_annotated_mgf_from_dat(Config *config, 
								   char *dat_list,
								   char *mzxml_list,
								   char *anns_file,
								   char *output_file);

void convert_dat_to_mgf(Config *config, 
						char *dat_list, 
						char *out_name, 
						char *out_dir,
						char *anns_file = NULL);

void find_best_similar_pairs(char *model_name, char *mgf1, char *mgf2, int num_pairs);
void find_self_similarity(char *model_name, char *mgf1, char *mgf2);
void find_similar_pairs_ditrib(char *model_name, char *mgf1, char *mgf2);
void find_homeometric_similarity_distrib(char *model_name, char *mgf1, char *mgf2);
void find_self_similarity_ranges(char *model_name, char *mgf1);
void peptide_distances();
void find_matches_similarity_distrib(char *model_name, char *mgf1, char *mgf2);

/*void create_annotated_mgf_from_mgf(Config *config, 
								   char *mgf_list,
								   char *anns_file,
								   char *output_file);*/


void test_sims();

int parse_single_MZXML_file_print_peaks(Config *config, 
										   string& mzxml_name, 
										   int the_scan);

void check_fs_for_missing_anns_and_test_sqs(Config *config,
											const vector<SingleSpectrumFile *>& ssfs,
											const FileManager& fm,
											void *pmcqsqs,
											char *anns_file);


#endif