hw97.ps

Mitchell的《机器学习〉随书源码

122 1297 y(among)13 b(20)i(p)q(eople\).)20 b(\(Hin)o(t:)e(lea)o(v)o(e)
12 b(learning)h(rate)h(and)g(momen)o(tum)9 b(at)14 b(0.3,)g(and)g(use)f
(20)h(hidden)122 1357 y(units\).)60 1459 y(6.)24 b(As)16
b(b)q(efore,)g(train)g(the)g(net)o(w)o(ork,)f(this)h(time)e(for)j(100)g
(ep)q(o)q(c)o(hs:)122 1540 y Fg(facetrain)22 b(-n)j(face.net)e(-t)i
(straighte)o(ven)p 1076 1540 V 15 w(train.list)d(-1)j(straighte)o(ven)p
1761 1540 V 15 w(test1.lis)o(t)122 1600 y(-2)g(straightev)o(en)p
512 1600 V 15 w(test2.list)d(-e)j(100)122 1681 y Fh(Y)l(ou)14
b(migh)o(t)e(b)q(e)h(w)o(ondering)h(wh)o(y)g(y)o(ou)f(are)h(only)f
(training)h(on)g(samples)f(from)f(a)i(limited)d(distribu-)122
1741 y(tion)16 b(\(the)h(\\straigh)o(t")g(images\).)k(The)16
b(essen)o(tial)g(reason)h(is)f(training)g(time.)k(If)c(y)o(ou)g(ha)o(v)
o(e)g(access)122 1802 y(to)e(a)h(v)o(ery)d(fast)j(mac)o(hine)d(\(an)o
(ything)i(slo)o(w)o(er)f(than)h(an)h(Alpha)e(or)i(Sun4)f(ma)o(y)f(b)q
(e)h(to)q(o)h(slo)o(w\),)f(then)122 1862 y(y)o(ou)g(are)h(w)o(elcome)d
(to)i(do)h(these)f(exp)q(erimen)o(ts)e(on)j(the)f(en)o(tire)f(set)i
(\(replace)e Fg(straight)f Fh(with)i Fg(all)122 1922
y Fh(in)i(the)g(command)e(ab)q(o)o(v)o(e.)21 b(Otherwise,)15
b(stic)o(k)g(to)i(the)f(\\straigh)o(t")h(images.)122
2003 y(The)g(di\013erence)g(b)q(et)o(w)o(een)f(the)h
Fg(straightrnd)p 1005 2003 V 15 w(*.list)e Fh(and)j(the)f
Fg(straighteve)o(n)p 1684 2003 V 15 w(*.list)f Fh(sets)122
2063 y(is)i(that)g(while)f(the)h(former)e(divides)h(the)h(images)f
(purely)g(randomly)g(among)h(the)f(training)h(and)122
2123 y(test)h(sets,)g(the)f(latter)h(ensures)g(a)g(relativ)o(ely)d(ev)o
(en)i(distribution)g(of)h(eac)o(h)g(individual's)e(images)122
2183 y(o)o(v)o(er)e(the)g(sets.)21 b(Because)15 b(w)o(e)h(ha)o(v)o(e)f
(only)g(7)h(or)g(8)g(\\straigh)o(t")h(images)e(p)q(er)h(individual,)e
(failure)h(to)122 2244 y(distribute)g(them)f(ev)o(enly)g(w)o(ould)h
(result)g(in)h(testing)f(our)h(net)o(w)o(ork)f(the)h(most)f(on)h(those)
g(faces)f(on)122 2304 y(whic)o(h)g(it)h(w)o(as)h(trained)f(the)g
(least.)60 2406 y(7.)24 b(Whic)o(h)18 b(parts)i(of)f(the)f(co)q(de)h(w)
o(as)h(it)e(necessary)h(to)g(mo)q(dify)f(this)g(time?)28
b(Ho)o(w)19 b(did)f(y)o(ou)h(enco)q(de)122 2466 y(the)c(outputs?)22
b(What)16 b(w)o(as)f(the)g(maxim)o(um)c(classi\014cation)j(accuracy)h
(ac)o(hiev)o(ed)e(on)j(the)f(training)122 2526 y(set?)27
b(Ho)o(w)18 b(man)o(y)f(ep)q(o)q(c)o(hs)i(did)f(it)g(tak)o(e)f(to)i
(reac)o(h)e(this)h(lev)o(el?)26 b(Ho)o(w)18 b(ab)q(out)h(for)g(the)f(v)
m(alidation)122 2586 y(and)f(test)f(set?)60 2688 y(8.)24
b(No)o(w)16 b(let's)f(tak)o(e)h(a)h(closer)e(lo)q(ok)i(at)g(whic)o(h)e
(images)g(the)h(net)g(ma)o(y)f(ha)o(v)o(e)g(failed)h(to)g(classify:)961
2844 y(4)p eop
%%Page: 5 5
5 4 bop 122 50 a Fg(facetrain)22 b(-n)j(face.net)e(-T)i(-1)g(straighte)
o(ven)p 1153 50 16 2 v 15 w(test1.lis)o(t)e(-2)i(straighte)o(ven)p
1838 50 V 15 w(test2.lis)o(t)122 131 y Fh(Do)17 b(there)e(seem)g(to)i
(b)q(e)f(an)o(y)g(particular)g(commonalities)d(b)q(et)o(w)o(een)i(the)h
(misclassi\014ed)f(images?)60 232 y(9.)24 b(Implem)o(en)o(t)12
b(a)k(p)q(ose)g(recognizer;)e(i.e.)20 b(implem)o(e)o(n)o(t)12
b(a)k(neural)f(net)g(whic)o(h,)g(when)g(giv)o(en)f(an)i(image)122
293 y(as)j(input,)f(indicates)f(whether)h(the)g(p)q(erson)g(in)g(the)g
(image)f(is)h(lo)q(oking)g(straigh)o(t)g(ahead,)h(up,)g(to)122
353 y(the)e(left,)f(or)i(to)f(the)g(righ)o(t.)24 b(Y)l(ou)17
b(will)g(also)g(need)g(to)h(implem)o(en)n(t)d(a)i(di\013eren)o(t)g
(output)h(enco)q(ding)122 413 y(for)13 b(this)g(task.)20
b(\(Hin)o(t:)f(lea)o(v)o(e)11 b(learning)h(rate)h(and)h(momen)o(tum)9
b(at)k(0.3,)g(and)h(use)f(6)g(hidden)f(units\).)35 515
y(10.)25 b(T)l(rain)16 b(the)g(net)o(w)o(ork)g(for)g(100)i(ep)q(o)q(c)o
(hs,)e(this)g(time)e(on)j(samples)e(dra)o(wn)i(from)e(all)h(of)g(the)g
(images:)122 596 y Fg(facetrain)22 b(-n)j(pose.net)e(-t)i(all)p
844 596 V 17 w(train.list)d(-1)j(all)p 1298 596 V 17
w(test1.list)122 656 y(-2)g(all)p 280 656 V 17 w(test2.list)d(-e)j(100)
122 737 y Fh(Since)15 b(the)h(p)q(ose-recognizing)h(net)o(w)o(ork)f
(should)g(ha)o(v)o(e)g(substan)o(tially)g(few)o(er)f(w)o(eigh)o(ts)h
(to)g(up)q(date)122 797 y(than)f(the)g(face-recognizing)f(net)o(w)o
(ork,)g(ev)o(en)g(those)h(of)g(y)o(ou)g(with)f(slo)o(w)h(mac)o(hines)e
(can)i(get)g(in)f(on)122 857 y(the)k(fun)g(of)g(using)g(all)g(of)g(the)
g(images.)25 b(In)18 b(this)g(case,)g(260)h(examples)d(are)i(in)g(the)f
(training)h(set,)122 917 y(140)f(examples)e(are)h(in)g(test1,)g(and)h
(193)g(are)f(in)g(test2.)35 1019 y(11.)25 b(Ho)o(w)19
b(did)g(y)o(ou)g(enco)q(de)g(y)o(our)g(outputs)h(this)f(time?)28
b(What)20 b(w)o(as)g(the)f(maxim)n(um)c(classi\014cation)122
1079 y(accuracy)21 b(ac)o(hiev)o(ed)e(on)j(the)e(training)i(set?)36
b(Ho)o(w)21 b(man)o(y)e(ep)q(o)q(c)o(hs)j(did)f(it)f(tak)o(e)h(to)g
(reac)o(h)g(this)122 1139 y(lev)o(el?)e(Ho)o(w)e(ab)q(out)g(for)g(eac)o
(h)e(test)h(set?)35 1241 y(12.)25 b(No)o(w,)15 b(try)h(taking)f(a)i(lo)
q(ok)f(at)g(ho)o(w)g(bac)o(kpropagation)h(tuned)e(the)h(w)o(eigh)o(ts)f
(of)h(the)g(hidden)f(units)122 1301 y(with)j(resp)q(ect)g(to)h(eac)o(h)
f(pixel.)26 b(First)18 b(t)o(yp)q(e)f Fg(make)25 b(hidtopgm)15
b Fh(to)k(compile)d(the)i(utilit)o(y)e(on)j(y)o(our)122
1362 y(system.)h(Then,)c(to)g(visualize)f(the)h(w)o(eigh)o(ts)g(of)g
(hidden)g(unit)g Fd(n)p Fh(,)g(t)o(yp)q(e:)122 1443 y
Fg(hidtopgm)23 b(pose.net)13 b Fd(image-\014lename)19
b Fg(32)25 b(30)15 b Fd(n)122 1523 y Fh(In)o(v)o(oking)c
Fg(xv)g Fh(on)h(the)g(image)f Fd(image-\014lename)j Fh(should)f(then)f
(displa)o(y)f(the)h(range)g(of)g(w)o(eigh)o(ts,)g(with)122
1584 y(the)k(lo)o(w)o(est)g(w)o(eigh)o(ts)g(mapp)q(ed)g(to)h(pixel)f(v)
m(alues)g(of)h(zero,)f(and)i(the)e(highest)h(mapp)q(ed)f(to)h(255.)23
b(If)122 1644 y(the)c(images)f(just)i(lo)q(ok)g(lik)o(e)d(noise,)j(try)
f(retraining)g(using)g Fg(facetrain)p 1491 1644 V 16
w(init0)e Fh(\(compile)g(with)122 1704 y Fg(make)24 b(facetrain)p
487 1704 V 15 w(init0)p Fh(\),)18 b(whic)o(h)g(initializes)f(the)h
(hidden)h(unit)g(w)o(eigh)o(ts)f(of)h(a)h(new)f(net)o(w)o(ork)122
1764 y(to)e(zero,)e(rather)h(than)h(random)f(v)m(alues.)35
1866 y(13.)25 b(Do)13 b(the)f(hidden)g(units)g(seem)e(to)j(w)o(eigh)o
(t)e(particular)h(regions)h(of)f(the)g(image)f(greater)i(than)f
(others?)122 1926 y(Do)17 b(particular)f(hidden)g(units)g(seem)e(to)j
(b)q(e)f(tuned)h(to)f(di\013eren)o(t)g(features)g(of)g(some)f(sort?)0
2098 y Fi(2.2.)55 b(P)n(art)20 b(I)r(I.)d(\(optional!\))0
2231 y Fh(No)o(w)g(that)g(y)o(ou)f(kno)o(w)h(y)o(our)g(w)o(a)o(y)f
(around)i Fg(facetrain)p Fh(,)13 b(it's)j(time)f(to)i(ha)o(v)o(e)f
(some)g(fun.)23 b(F)l(orm)15 b(a)i(team)0 2292 y(with)k(one)g(or)g(t)o
(w)o(o)f(other)h(studen)o(ts,)h(and)f(pic)o(k)f(some)f(in)o(teresting)h
(topic)g(of)h(y)o(our)g(o)o(wn)g(c)o(hoice)f({)h(b)q(e)0
2352 y(creativ)o(e!!)j(Run)18 b(some)f(exp)q(erimen)o(ts,)f(and)i
(prepare)g(a)h(short)f(write-up)g(of)g(what)h(y)o(our)f(group's)h
(idea,)0 2412 y(exp)q(erimen)o(tal)14 b(results,)j(and)h(an)o(y)f
(conclusions)h(y)o(ou)f(dra)o(w)h(\(a)f(few)g(pages)h(should)g(b)q(e)g
(su\016cien)o(t\).)23 b(F)l(or)0 2472 y(this)16 b(part)h(of)f(the)g
(assignmen)o(t,)f(y)o(our)h(group)h(can)g(turn)f(in)g(a)h(single)e
(group)j(writeup.)62 2574 y(Some)e(p)q(ossibilities)g(for)h(exp)q
(erimen)o(tation)d(are)j(\(but)f(please)h(don't)g(let)e(this)i(list)f
(limit)e(y)o(ou)j(in)f(an)o(y)0 2634 y(w)o(a)o(y)g(if)g(y)o(ou)g(w)o
(an)o(t)g(to)h(try)f(something)f(else\):)961 2844 y(5)p
eop
%%Page: 6 6
6 5 bop 73 50 a Ff(\017)24 b Fh(Use)c(the)h(output)h(of)f(the)g(p)q
(ose)g(recognizer)g(as)g(input)g(to)g(the)g(face)g(recognizer,)f(and)i
(see)f(ho)o(w)122 110 y(this)16 b(a\013ects)g(p)q(erformance.)k(T)l(o)d
(do)f(this,)g(y)o(ou)g(will)f(need)g(to)i(add)g(a)f(mec)o(hanism)d(for)
j(sa)o(ving)g(the)122 170 y(output)e(units)g(of)f(the)h(p)q(ose)g
(recognizer)f(and)h(a)g(mec)o(hanism)c(for)k(loading)g(this)f(data)i
(in)o(to)e(the)g(face)122 230 y(recognizer.)73 332 y
Ff(\017)24 b Fh(Learn)18 b(the)f Fd(lo)n(c)n(ation)h
Fh(of)g(some)e(feature)i(in)f(the)g(image,)f(suc)o(h)h(as)i(ey)o(es.)24
b(Y)l(ou)17 b(can)h(use)f Fg(xv)g Fh(to)h(tell)122 392
y(y)o(ou)e(the)g(co)q(ordinates)g(of)h(the)e(feature)h(in)g(question)g
(for)g(eac)o(h)f(image,)g(whic)o(h)g(y)o(ou)h(can)g(then)g(use)122
452 y(as)h(y)o(our)f(target)h(v)m(alues.)73 554 y Ff(\017)24
b Fh(T)l(ak)o(e)16 b(a)g(lo)q(ok)h(at)g(the)f(additional)g(data)h(from)
e(an)i(earlier)e(y)o(ear's)h(class)g(in)122 635 y Fg(/afs/cs.cm)o(u.e)o
(du)o(/pr)o(oje)o(ct)o(/th)o(eo-)o(8/m)o(l9)o(4fa)o(ces)o
Fh(.)122 716 y(What)j(tec)o(hniques)e(can)i(y)o(ou)f(emplo)o(y)e(to)j
(train)f(nets)h(to)g(generalize)e(b)q(etter)h(from)f(one)i(dataset)122
776 y(to)e(the)f(other?)73 878 y Ff(\017)24 b Fh(Ho)o(w)14
b(do)h(nets)f(p)q(erform)g(if)g(trained)g(on)h(more)e(than)i(one)f
(concept)g(at)h(once?)21 b(Do)15 b(represen)o(tations)122
938 y(formed)g(for)h(m)o(ultiple)d(concepts)j(in)o(terfere)e(with)i
(eac)o(h)g(other)g(in)g(the)f(hidden)h(la)o(y)o(er,)e(or)j(p)q(erhaps)
122 998 y(augmen)o(t)e(eac)o(h)h(other?)73 1100 y Ff(\017)24
b Fh(Use)19 b(the)h(image)f(pac)o(k)m(age,)i(w)o(eigh)o(t)e
(visualization)g(utilit)o(y)l(,)g(and/or)i(an)o(ything)f(else)g(y)o(ou)
g(migh)o(t)122 1160 y(ha)o(v)o(e)i(a)o(v)m(ailable)g(to)i(try)e(to)i
(understand)f(b)q(etter)g(what)h(the)e(net)o(w)o(ork)h(has)h(actually)e
(learned.)122 1220 y(Using)c(this)g(information,)f(what)i(do)g(y)o(ou)f
(think)g(the)g(net)o(w)o(ork)f(is)h(learning?)28 b(Can)19
b(y)o(ou)f(exploit)122 1281 y(this)e(information)f(to)i(impro)o(v)o(e)c
(generalization?)73 1382 y Ff(\017)24 b Fh(Change)17
b(the)f(input)g(or)h(output)g(enco)q(dings)f(to)h(try)f(to)h(impro)o(v)
o(e)c(generalization)j(accuracy)l(.)73 1484 y Ff(\017)24
b Fh(V)l(ary)d(the)g(n)o(um)o(b)q(er)f(of)h(hidden)g(units,)h(the)f(n)o
(um)o(b)q(er)f(of)h(training)h(examples,)e(the)h(n)o(um)o(b)q(er)f(of)
122 1544 y(ep)q(o)q(c)o(hs,)e(the)f(momen)o(tum)c(and)18
b(learning)f(rate,)g(or)h(whatev)o(er)f(else)g(y)o(ou)g(w)o(an)o(t)g
(to)h(try)l(,)f(with)g(the)122 1604 y(goal)i(of)g(getting)f(the)g
(greatest)h(p)q(ossible)f(discrepancy)g(b)q(et)o(w)o(een)f(train)i(and)
g(test)f(set)g(accuracy)122 1665 y(\(i.e.,)c(ho)o(w)j(badly)f(can)h(y)o
(ou)f(mak)o(e)e(the)i(net)o(w)o(ork)g(o)o(v)o(er\014t\),)f(and)i(the)f
(smallest)f(p)q(ossible)h(discrep-)122 1725 y(ancy)g(\(i.e.,)e(what)j
(is)f(the)g(b)q(est)h(p)q(erformance)e(y)o(ou)h(can)g(ac)o(hiev)o(e\).)
0 1902 y Fi(3.)56 b(Do)r(cumen)n(tation)0 2053 y Fh(The)16
b(co)q(de)h(for)f(this)g(assignmen)o(t)g(is)g(brok)o(en)f(in)o(to)h
(sev)o(eral)g(mo)q(dules:)73 2208 y Ff(\017)24 b Fg(pgmimage.c)o
Fh(,)10 b Fg(pgmimage.)o(h)p Fh(:)16 b(the)c(image)e(pac)o(k)m(age.)20
b(Supp)q(orts)14 b(read/write)d(of)i(PGM)f(image)e(\014les)122
2268 y(and)21 b(pixel)e(access/assignmen)o(t.)33 b(Pro)o(vides)20
b(an)h Fg(IMAGE)e Fh(data)i(structure,)g(and)g(an)g Fg(IMAGELIST)122
2329 y Fh(data)f(structure)f(\(an)h(arra)o(y)f(of)h(p)q(oin)o(ters)f
(to)h(images;)f(useful)g(when)g(handling)h(man)o(y)d(images\).)122
2389 y Fi(Y)-5 b(ou)22 b(will)e(not)h(need)g(to)g(mo)r(dify)e(an)n(y)j
(co)r(de)e(in)h(this)g(mo)r(dule)e(to)i(complete)d(the)j(as-)122
2449 y(signmen)n(t.)73 2551 y Ff(\017)j Fg(backprop.c)o
Fh(,)d Fg(backprop.)o(h)p Fh(:)30 b(the)22 b(neural)g(net)o(w)o(ork)f
(pac)o(k)m(age.)40 b(Supp)q(orts)23 b(three-la)o(y)o(er)e(fully-)122
2611 y(connected)15 b(feedforw)o(ard)g(net)o(w)o(orks,)g(using)h(the)f
(bac)o(kpropagation)i(algorithm)e(for)h(w)o(eigh)o(t)e(tun-)122
2671 y(ing.)21 b(Pro)o(vides)16 b(high)g(lev)o(el)e(routines)i(for)g
(creating,)f(training,)h(and)g(using)h(net)o(w)o(orks.)j
Fi(Y)-5 b(ou)20 b(will)122 2731 y(not)f(need)f(to)g(mo)r(dify)e(an)n(y)
k(co)r(de)d(in)i(this)f(mo)r(dule)e(to)j(complete)d(the)i(assignmen)n
(t.)961 2844 y Fh(6)p eop
%%Page: 7 7
7 6 bop 73 50 a Ff(\017)24 b Fg(imagenet.c)o Fh(:)16
b(in)o(terface)9 b(routines)i(for)g(loading)h(images)d(in)o(to)i(the)f
(input)h(units)g(of)g(a)g(net)o(w)o(ork,)g(and)122 110
y(setting)g(up)g(target)h(v)o(ectors)e(for)i(training.)19
b(Y)l(ou)12 b(will)e(need)g(to)i(mo)q(dify)e(the)h(routine)g
Fg(load)p 1766 110 16 2 v 17 w(target)p Fh(,)122 170
y(when)16 b(implem)o(en)n(ting)d(the)j(face)f(recognizer)g(and)h(the)g
(p)q(ose)g(recognizer,)f(to)h(set)f(up)h(appropriate)122
230 y(target)h(v)o(ectors)e(for)i(the)f(output)h(enco)q(dings)f(y)o(ou)
g(c)o(ho)q(ose.)73 332 y Ff(\017)24 b Fg(facetrain.)o(c)p
Fh(:)17 b(the)c(top-lev)o(el)f(program)h(whic)o(h)g(uses)h(all)f(of)g
(the)h(mo)q(dules)e(ab)