kdd.c

Scalable k-means software and test datasets This package (a Unix tar file, gzipped) contains the sou

{ "ADATE_13", 0, 4, 9502, 9507 },
{ "ADATE_14", 0, 3, 9504, 9506 },
{ "ADATE_15", 0, 1, 9504, 9504 },
{ "ADATE_16", 0, 3, 9502, 9504 },
{ "ADATE_17", 0, 3, 9501, 9503 },
{ "ADATE_18", 0, 4, 9409, 9508 },
{ "ADATE_19", 0, 4, 9409, 9411 },
{ "ADATE_20", 0, 2, 9411, 9412 },
{ "ADATE_21", 0, 2, 9409, 9410 },
{ "ADATE_22", 0, 4, 9408, 9506 },
{ "ADATE_23", 0, 4, 9313, 9407 },
{ "ADATE_24", 0, 2, 9405, 9406 },
{ "RFA_2", 0, 0 },
{ "RFA_3", 0, 0 },
{ "RFA_4", 0, 0 },
{ "RFA_5", 0, 0 },
{ "RFA_6", 0, 0 },
{ "RFA_7", 0, 0 },
{ "RFA_8", 0, 0 },
{ "RFA_9", 0, 0 },
{ "RFA_10", 0, 0 },
{ "RFA_11", 0, 0 },
{ "RFA_12", 0, 0 },
{ "RFA_13", 0, 0 },
{ "RFA_14", 0, 0 },
{ "RFA_15", 0, 0 },
{ "RFA_16", 0, 0 },
{ "RFA_17", 0, 0 },
{ "RFA_18", 0, 0 },
{ "RFA_19", 0, 0 },
{ "RFA_20", 0, 0 },
{ "RFA_21", 0, 0 },
{ "RFA_22", 0, 0 },
{ "RFA_23", 0, 0 },
{ "RFA_24", 0, 0 },
{ "CARDPROM", 3, 6, 1, 61 },
{ "MAXADATE", 4, 4, 9608, 9702 },
{ "NUMPROM", 3, 4, 4, 195 },
{ "CARDPM12", 0, 4, 0, 19 },
{ "NUMPRM12", 0, 4, 1, 78 },
{ "RDATE_3", 0, 4, 9605, 9806 },
{ "RDATE_4", 0, 4, 9510, 9804 },
{ "RDATE_5", 0, 4, 9604, 9803 },
{ "RDATE_6", 0, 4, 9510, 9805 },
{ "RDATE_7", 0, 4, 9512, 9610 },
{ "RDATE_8", 0, 4, 9511, 9806 },
{ "RDATE_9", 0, 4, 9509, 9609 },
{ "RDATE_10", 0, 4, 9510, 9806 },
{ "RDATE_11", 0, 4, 9509, 9805 },
{ "RDATE_12", 0, 4, 9509, 9806 },
{ "RDATE_13", 0, 4, 9502, 9603 },
{ "RDATE_14", 0, 4, 9406, 9603 },
{ "RDATE_15", 0, 4, 9412, 9603 },
{ "RDATE_16", 0, 4, 9411, 9805 },
{ "RDATE_17", 0, 4, 9502, 9512 },
{ "RDATE_18", 0, 4, 9412, 9601 },
{ "RDATE_19", 0, 4, 9409, 9509 },
{ "RDATE_20", 0, 4, 9411, 9508 },
{ "RDATE_21", 0, 4, 9409, 9508 },
{ "RDATE_22", 0, 4, 9409, 9510 },
{ "RDATE_23", 0, 4, 9309, 9507 },
{ "RDATE_24", 0, 4, 9309, 9504 },
{ "RAMNT_3", 3, 50, 0, 50 },
{ "RAMNT_4", 3, 50, 0, 100 },
{ "RAMNT_5", 3, 50, 0, 50 },
{ "RAMNT_6", 3, 50, 0, 100 },
{ "RAMNT_7", 3, 50, 0, 250 },
{ "RAMNT_8", 3, 50, 0, 500 },
{ "RAMNT_9", 3, 50, 0, 1000 },
{ "RAMNT_10", 3, 50, 0, 500 },
{ "RAMNT_11", 3, 50, 0, 300 },
{ "RAMNT_12", 3, 50, 0, 300 },
{ "RAMNT_13", 3, 50, 0, 500 },
{ "RAMNT_14", 3, 50, 0, 200 },
{ "RAMNT_15", 3, 50, 0, 300 },
{ "RAMNT_16", 3, 50, 0, 500 },
{ "RAMNT_17", 3, 50, 0, 500 },
{ "RAMNT_18", 3, 50, 0, 1000 },
{ "RAMNT_19", 3, 50, 0, 970 },
{ "RAMNT_20", 3, 50, 0, 250 },
{ "RAMNT_21", 3, 50, 0, 300 },
{ "RAMNT_22", 3, 50, 0, 300 },
{ "RAMNT_23", 3, 50, 0, 200 },
{ "RAMNT_24", 3, 50, 0, 225 },
{ "RAMNTALL", 3, 50, 0, 9485 },
{ "NGIFTALL", 3, 10, 0, 237 },
{ "CARDGIFT", 3, 4, 0, 41 },
{ "MINRAMNT", 3, 10, 0, 1000 },
{ "MINRDATE", 4, 4, 7506, 9702 },
{ "MAXRAMNT", 3, 10, 0, 5000 },
{ "MAXRDATE", 4, 4, 7510, 9702 },
{ "LASTGIFT", 3, 10, 0, 1000 },
{ "LASTDATE", 4, 4, 9503, 9702 },
{ "FISTDATE", 4, 4, 0, 9603 },
{ "NEXTDATE", 4, 4, 7211, 9702 },
{ "TIMELAG", 3, 10, 0, 1088 },
{ "AVGGIFT", 3, 10, 0, 1000 },
{ "CONTROLN", 0, 0 },
{ "TARGET_B", 0, 0 },
{ "TARGET_D", 0, 2 },
{ "HPHONE_D", 0, 0 },
//{ "RFA_2R", 6, 6, 0, 0, "F", "N", "L", "A", "I", "S" },
//{ "RFA_2F", 6, 4, 0, 0, "1", "2", "3", "4" },
//{ "RFA_2A", 6, 7, 0, 0, "A", "B", "C", "D", "E", "F", "G" },
{ "RFA_2R", 0, 6, 0, 0, "F", "N", "L", "A", "I", "S" },
{ "RFA_2F", 6, 4, 0, 0, "1", "2", "3", "4" },
{ "RFA_2A", 6, 4, 0, 0, "D", "E", "F", "G" },
{ "MDMAUD_R", 0, 0 },
{ "MDMAUD_F", 0, 0 },
{ "MDMAUD_A", 0, 0 },
{ "CLUSTER2", 0, 0 },
{ "GEOCODE2", 0, 0 }
};

//static int NumFields = 10;
static int NumFields = 481;

static int Dimensions;
static int Counter;

// Vector to output converted data.
static float *Data;
static float *SumData;

// File pointers for input and output data files.
FILE *DataInFp;
FILE *DataTmpFp;
FILE *DataOutFp;

// Allocate vectors used to preocess data records.
int Setup(void)
{
        int i;
	Dimensions = 0;
        Data = 0;
	SumData = 0;

        for (i = 0; i < NumFields; i++) {
		if (Fields[i].type == 0) continue;
		if (Fields[i].type == 3 || Fields[i].type == 4) {
			Dimensions++;
		} else {
                	Dimensions += Fields[i].cnt;
		}
	}
	printf("Dimensions: %d\n", Dimensions);

        // Allocate Data and sum Data array.

        if(!(Data = (float*) malloc(sizeof(float) * Dimensions)))
                return 1;

	if(!(SumData = (float*) malloc(sizeof(float) * Dimensions))) {
		free(Data);
		Data = 0;
		return 1;
	}
		
        for (i = 0; i < Dimensions; i++) {
		SumData[i] = 0;
		Data[i] = 0;
	}

        return 0;
}


// Convert a date from YYMM to months.
int toMonth(int date)
{
        int year, month;
        month = date;
        year = (date >> 2) << 2;
        month -= year;
        return ((year * 12) / 100) + month;
}


// Process the input data file.
int Process(void)
{
        int i, k;
        char *v;
        char line[4096];
        char token[32];
        char *tPtr;
        int field;
        int iIdx, oIdx;
        double value;
//		float value;
        int next;
        int m;

	while(1) {

        	/* Number of records to read */
        	//if (Counter >= 20)
                	//return 1;

        	iIdx = 0;
        	field = 0;

        	/* Read in data record and parse into tokens delimited by a comma */
        	if ((fgets(line, 4096, DataInFp)) == NULL)
                	return 0;

        	tPtr = line;

		for (i = 0; i < Dimensions; i++)
			Data[i] = 0;

        	while (field <= NumFields) {
                	field++;
                	next = strcspn(tPtr, ",");
                	strncpy(token, tPtr, next);
                	token[next] = '\0';
                	tPtr += ++next;

                	switch (Fields[field].type) {
                        	case 0:		// Do not use this field.
                                	break;
                        	case 1:         // Flag
                                	if (token[0] == 'X') {
                                        	Data[iIdx] = 1;
                                	} else if (token[0] == ' ') {
						Data[iIdx] = 0;
					} else {
						Data[iIdx] = -1;
					}
					iIdx++;
                                	break;
                        	case 3:         // Numeric
//			  		value = (float) atof(token);
					if(sscanf(token, "%lf", &value) == 1)
					{	//printf("%d '%s' %f\n", field, token, value);
						Data[iIdx] = value;
					}
					else
					{
						if(field >= 435 && field <= 456)
							Data[iIdx] = 0;
						else
							Data[iIdx] = -1;
					}
					iIdx++;
					break;
					/*
					} else {
						Data[iIdx] = value;
                                	}
                                	break;
					*/
                        	case 4:         // Date
					
					if (sscanf(token, "%lf", &value) != 1) {
						Data[iIdx] = -1;
					} else {
//printf("'%s' %f\n", token, value);
                                		value = toMonth(value - Fields[field].min);
						Data[iIdx] = value;
					}
					
//					value = 0;
//					value = (double) atof(token);
//printf("'%s' %lf\n", token, value);
//                                        Data[iIdx] = toMonth(value);
					iIdx++;
                                	break;
                        	case 5:         // Yes or No
                                	if (token[0] == 'Y') {
                                        	Data[iIdx] = 1;
                                	} else if (token[0] == 'N') {
						Data[iIdx] = 0;
					} else {
						Data[iIdx] = -1;
					}
					iIdx++;
                                	break;
				case 6:		// String matches
//					printf("%s %d %d %s\n", Fields[field].name, field, iIdx, token);
					for (m = 0; m < Fields[field].cnt; m++) {
						if (strcmp(token, Fields[field].values[m]) == 0) {
							Data[iIdx+m] = 1;
							break;
						}
					}
					if (m == Fields[field].cnt) {
						for (m = 0; m < Fields[field].cnt; m++) {
                                                	Data[iIdx+m] = -1;
                                                }
                                        }
					iIdx += Fields[field].cnt;
					break;
                	}
        	}

        	/* Display input and output vectors, testing only */
		for (i = 0; i < Dimensions; i++) {
			if ((fprintf(DataOutFp, "%f ", Data[i])) == 0) {
				printf("Error writing data1.");
				exit(1);
			}
			if (Data[i] != -1) {
				SumData[i] += Data[i];
			}
		}
		if ((fprintf(DataOutFp, "\n")) == 0) {
                	printf("Error writing data2.");
                        exit(1);
                }

        	Counter++;
	}

}


void Process2(void)
{
	int i, j;
	float tmp;

	printf("Records: %d\n", Counter);
	Counter--;

	while(Counter--)
	{
		for (i = 0; i < Dimensions; i++) {
			if ((fscanf(DataTmpFp, "%f", &tmp)) == 0) {
				printf("Error writing data3.");
                        	return;
                	}
			if (tmp != -1) {
				if (tmp > 0)
				   tmp /= SumData[i];
			}
	
			if ((fprintf(DataOutFp, "%f ", tmp)) == 0) {
                           	printf("Error writing data4.\n");
                              	exit(1);
                        }
		}
		if ((fprintf(DataOutFp, "\n")) == 0) {
                        printf("Error writing data5.\n");
                        exit(1);
                }
	}

}


// Free memory buffers.
void FreeVectors(void)
{
        if (Data) {
        	free(Data);
                Data = 0;
        }
	if (SumData) {
		free(SumData);
		SumData = 0;
	}
}


// Main routine.
int main(int argc, char *argv[])
{
        Counter = 0;

        Setup();

        // Open input data file.
        if ((DataInFp = fopen("cup98lrnB.txt", "r")) == NULL) {
                perror("Unable to open data file.");
                exit(1);
        }

        // Open temp data file.
        if ((DataOutFp = fopen("cup98.dat", "w")) == NULL) {
                perror("Unable to open data file.");
                exit(1);
        }

	Process();
	fclose(DataInFp);
	fclose(DataOutFp);

/*        // Open temp data file.
        if ((DataTmpFp = fopen("cup98.tmp", "r")) == NULL) {
                perror("Unable to open data file.");
                exit(1);
        }

        // Open output data file.
        if ((DataOutFp = fopen("cup98.dat", "w")) == NULL) {
                perror("Unable to open data file.");
                exit(1);
        }

        Process2();
        fclose(DataTmpFp);
        fclose(DataOutFp);
*/
        FreeVectors();
}

/* End of file kdd.c */