trecqrels.txt

Lucene a java open-source SearchEngine Framework

# -----------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -----------------------------------------------------------------------

# ------------------------------------------------------------
# Format:
#
#       qnum   0   doc-name     is-relevant
#
#
# The origin of this file was created using 
# utils.QualityQueriesFinder, so all queries 
# would have perfect 1.0 for all meassures.
#
# To make it suitable for testing it was modified
# for some queries, depending on m = qnum % 8
# m==0: avg_precision and recall are hurt, by marking fake docs as relevant
# m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs
# m==2: all precision, precision_at_n and recall are hurt.
# m>=3: these queries remain perfect
# ------------------------------------------------------------

# --- m==0: avg_precision and recall are hurt, by marking fake docs as relevant

0 	 0 	 fakedoc1             	 1
0 	 0 	 fakedoc2             	 1
0 	 0 	 fakedoc3             	 1
0 	 0 	 fakedoc4             	 1

0 	 0 	 doc18211             	 1
0 	 0 	 doc20192             	 1
0 	 0 	 doc7401              	 1
0 	 0 	 doc11285             	 1
0 	 0 	 doc20647             	 1
0 	 0 	 doc3057              	 1
0 	 0 	 doc12431             	 1
0 	 0 	 doc4989              	 1
0 	 0 	 doc17324             	 1
0 	 0 	 doc4030              	 1
0 	 0 	 doc4290              	 1
0 	 0 	 doc3462              	 1
0 	 0 	 doc15313             	 1
0 	 0 	 doc10303             	 1
0 	 0 	 doc1893              	 1
0 	 0 	 doc5008              	 1
0 	 0 	 doc14634             	 1
0 	 0 	 doc5471              	 1
0 	 0 	 doc17904             	 1
0 	 0 	 doc7168              	 1
0 	 0 	 doc21275             	 1
0 	 0 	 doc9011              	 1
0 	 0 	 doc17546             	 1
0 	 0 	 doc9102              	 1
0 	 0 	 doc13199             	 1

# --- m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs

1 	 0 	 doc9857              	 0
1 	 0 	 doc16846             	 1
1 	 0 	 doc4320              	 1
1 	 0 	 doc9501              	 0
1 	 0 	 doc10159             	 1
1 	 0 	 doc16642             	 1
1 	 0 	 doc17536             	 0
1 	 0 	 doc17571             	 1
1 	 0 	 doc18728             	 1
1 	 0 	 doc18828             	 1
1 	 0 	 doc19108             	 0
1 	 0 	 doc9940              	 1
1 	 0 	 doc11852             	 1
1 	 0 	 doc7430              	 0
1 	 0 	 doc19162             	 1
1 	 0 	 doc1743              	 1
1 	 0 	 doc2137              	 1
1 	 0 	 doc7611              	 1
1 	 0 	 doc8072              	 1
1 	 0 	 doc12764             	 1
1 	 0 	 doc2593              	 1
1 	 0 	 doc11088             	 1
1 	 0 	 doc931               	 1
1 	 0 	 doc7673              	 1
1 	 0 	 doc12941             	 1
1 	 0 	 doc11797             	 1
1 	 0 	 doc11831             	 1
1 	 0 	 doc13162             	 1
1 	 0 	 doc4423              	 1
1 	 0 	 doc5217              	 1

# ---- m==2: all precision, precision_at_n and recall are hurt.

2 	 0 	 fakedoc1             	 1
2 	 0 	 fakedoc2             	 1
2 	 0 	 fakedoc3             	 1
2 	 0 	 fakedoc4             	 1

2 	 0 	 doc3137              	 0
2 	 0 	 doc7142              	 0
2 	 0 	 doc13667             	 0
2 	 0 	 doc13171             	 0
2 	 0 	 doc13372             	 1
2 	 0 	 doc21415             	 1
2 	 0 	 doc16298             	 1
2 	 0 	 doc14957             	 1
2 	 0 	 doc153               	 1
2 	 0 	 doc16092             	 1
2 	 0 	 doc16096             	 1
2 	 0 	 doc21303             	 1
2 	 0 	 doc18681             	 1
2 	 0 	 doc20756             	 1
2 	 0 	 doc355               	 1
2 	 0 	 doc13395             	 1
2 	 0 	 doc5009              	 1
2 	 0 	 doc17164             	 1
2 	 0 	 doc13162             	 1
2 	 0 	 doc11757             	 1
2 	 0 	 doc9637              	 1
2 	 0 	 doc18087             	 1
2 	 0 	 doc4593              	 1
2 	 0 	 doc4677              	 1
2 	 0 	 doc20865             	 1
2 	 0 	 doc8556              	 1
2 	 0 	 doc2578              	 1
2 	 0 	 doc1163              	 1
2 	 0 	 doc3797              	 1
2 	 0 	 doc11094             	 1


3 	 0 	 doc19578             	 1
3 	 0 	 doc14860             	 1
3 	 0 	 doc7235              	 1
3 	 0 	 doc20590             	 1
3 	 0 	 doc17933             	 1
3 	 0 	 doc9384              	 1
3 	 0 	 doc10783             	 1
3 	 0 	 doc1963              	 1
3 	 0 	 doc18356             	 1
3 	 0 	 doc13254             	 1
3 	 0 	 doc18402             	 1
3 	 0 	 doc15241             	 1
3 	 0 	 doc3303              	 1
3 	 0 	 doc8868              	 1
3 	 0 	 doc18520             	 1
3 	 0 	 doc4650              	 1
3 	 0 	 doc4727              	 1
3 	 0 	 doc21518             	 1
3 	 0 	 doc5060              	 1
3 	 0 	 doc7587              	 1
3 	 0 	 doc2990              	 1
3 	 0 	 doc8042              	 1
3 	 0 	 doc6304              	 1
3 	 0 	 doc13223             	 1
3 	 0 	 doc1964              	 1
3 	 0 	 doc10597             	 1
3 	 0 	 doc21023             	 1
3 	 0 	 doc19057             	 1
3 	 0 	 doc14948             	 1
3 	 0 	 doc9692              	 1


4 	 0 	 doc2534              	 1
4 	 0 	 doc21388             	 1
4 	 0 	 doc20923             	 1
4 	 0 	 doc11547             	 1
4 	 0 	 doc19755             	 1
4 	 0 	 doc3793              	 1
4 	 0 	 doc6714              	 1
4 	 0 	 doc12722             	 1
4 	 0 	 doc5552              	 1
4 	 0 	 doc6810              	 1
4 	 0 	 doc16953             	 1
4 	 0 	 doc2527              	 1
4 	 0 	 doc5361              	 1
4 	 0 	 doc12353             	 1
4 	 0 	 doc7308              	 1
4 	 0 	 doc3836              	 1
4 	 0 	 doc2293              	 1
4 	 0 	 doc7348              	 1
4 	 0 	 doc17119             	 1
4 	 0 	 doc19331             	 1
4 	 0 	 doc3411              	 1
4 	 0 	 doc14643             	 1
4 	 0 	 doc9058              	 1
4 	 0 	 doc11099             	 1
4 	 0 	 doc12485             	 1
4 	 0 	 doc16432             	 1
4 	 0 	 doc10047             	 1
4 	 0 	 doc13788             	 1
4 	 0 	 doc117               	 1
4 	 0 	 doc638               	 1



5 	 0 	 doc169               	 1
5 	 0 	 doc13181             	 1
5 	 0 	 doc4350              	 1
5 	 0 	 doc10242             	 1
5 	 0 	 doc955               	 1
5 	 0 	 doc5389              	 1
5 	 0 	 doc17122             	 1
5 	 0 	 doc17417             	 1
5 	 0 	 doc12199             	 1
5 	 0 	 doc6918              	 1
5 	 0 	 doc3857              	 1
5 	 0 	 doc2981              	 1
5 	 0 	 doc10639             	 1
5 	 0 	 doc10478             	 1
5 	 0 	 doc8573              	 1
5 	 0 	 doc9197              	 1
5 	 0 	 doc9298              	 1
5 	 0 	 doc2492              	 1
5 	 0 	 doc10262             	 1
5 	 0 	 doc5180              	 1
5 	 0 	 doc11758             	 1
5 	 0 	 doc4065              	 1
5 	 0 	 doc9124              	 1
5 	 0 	 doc11528             	 1
5 	 0 	 doc18879             	 1
5 	 0 	 doc17864             	 1
5 	 0 	 doc3204              	 1
5 	 0 	 doc12157             	 1
5 	 0 	 doc4496              	 1
5 	 0 	 doc20190             	 1



6 	 0 	 doc9507              	 1
6 	 0 	 doc15630             	 1
6 	 0 	 doc8469              	 1
6 	 0 	 doc11918             	 1
6 	 0 	 doc20482             	 1
6 	 0 	 doc20158             	 1
6 	 0 	 doc19831             	 1
6 	 0 	 doc8296              	 1
6 	 0 	 doc8930              	 1
6 	 0 	 doc16460             	 1
6 	 0 	 doc2577              	 1
6 	 0 	 doc15476             	 1
6 	 0 	 doc1767              	 1
6 	 0 	 doc689               	 1
6 	 0 	 doc16606             	 1
6 	 0 	 doc6149              	 1
6 	 0 	 doc18691             	 1
6 	 0 	 doc2208              	 1
6 	 0 	 doc3592              	 1
6 	 0 	 doc11199             	 1
6 	 0 	 doc16329             	 1
6 	 0 	 doc6007              	 1
6 	 0 	 doc15231             	 1
6 	 0 	 doc20622             	 1
6 	 0 	 doc21468             	 1
6 	 0 	 doc12230             	 1
6 	 0 	 doc5723              	 1
6 	 0 	 doc8120              	 1
6 	 0 	 doc8668              	 1
6 	 0 	 doc303               	 1




7 	 0 	 doc7728              	 1
7 	 0 	 doc7693              	 1
7 	 0 	 doc21088             	 1
7 	 0 	 doc5017              	 1
7 	 0 	 doc10807             	 1
7 	 0 	 doc16204             	 1
7 	 0 	 doc2233              	 1
7 	 0 	 doc3632              	 1
7 	 0 	 doc4719              	 1
7 	 0 	 doc6477              	 1
7 	 0 	 doc6502              	 1
7 	 0 	 doc6709              	 1
7 	 0 	 doc7710              	 1
7 	 0 	 doc9193              	 1
7 	 0 	 doc9309              	 1
7 	 0 	 doc9789              	 1
7 	 0 	 doc10971             	 1
7 	 0 	 doc18059             	 1
7 	 0 	 doc19906             	 1
7 	 0 	 doc20089             	 1
7 	 0 	 doc20102             	 1
7 	 0 	 doc21040             	 1
7 	 0 	 doc21153             	 1
7 	 0 	 doc9147              	 1
7 	 0 	 doc9930              	 1
7 	 0 	 doc19763             	 1
7 	 0 	 doc1559              	 1
7 	 0 	 doc21248             	 1
7 	 0 	 doc17945             	 1
7 	 0 	 doc526               	 1


# --- m==0: avg_precision and recall are hurt, by marking fake docs as relevant

8 	 0 	 fakedoc1             	 1
8 	 0 	 fakedoc2             	 1
8 	 0 	 fakedoc3             	 1
8 	 0 	 fakedoc4             	 1

8 	 0 	 doc16299             	 1
8 	 0 	 doc1662              	 1
8 	 0 	 doc4585              	 1
8 	 0 	 doc12315             	 1
8 	 0 	 doc16266             	 1
8 	 0 	 doc13136             	 1
8 	 0 	 doc19212             	 1
8 	 0 	 doc7086              	 1
8 	 0 	 doc7062              	 1
8 	 0 	 doc6134              	 1
8 	 0 	 doc13953             	 1
8 	 0 	 doc16264             	 1
8 	 0 	 doc2494              	 1
8 	 0 	 doc10636             	 1
8 	 0 	 doc10894             	 1
8 	 0 	 doc6844              	 1
8 	 0 	 doc674               	 1
8 	 0 	 doc13520             	 1
8 	 0 	 doc344               	 1
8 	 0 	 doc2896              	 1
8 	 0 	 doc11871             	 1
8 	 0 	 doc1862              	 1
8 	 0 	 doc16728             	 1
8 	 0 	 doc10308             	 1
8 	 0 	 doc2227              	 1
8 	 0 	 doc13167             	 1
8 	 0 	 doc20607             	 1
8 	 0 	 doc9670              	 1
8 	 0 	 doc1566              	 1
8 	 0 	 doc17885             	 1


# ---- m==1: precision_at_n and avg_precision are hurt, by unmarking relevant docs


9 	 0 	 doc1990              	 0
9 	 0 	 doc9342              	 1
9 	 0 	 doc19427             	 1
9 	 0 	 doc12432             	 0
9 	 0 	 doc13480             	 1
9 	 0 	 doc3322              	 1
9 	 0 	 doc16044             	 1
9 	 0 	 doc266               	 0
9 	 0 	 doc3437              	 1
9 	 0 	 doc5370              	 1
9 	 0 	 doc10314             	 1
9 	 0 	 doc4892              	 1
9 	 0 	 doc5763              	 0
9 	 0 	 doc14045             	 1
9 	 0 	 doc1090              	 1
9 	 0 	 doc7437              	 1