swingapplet.java

用java写的一个强化学习程序

		pane.add(instructLabel);
		pane.add(usageLabel);
		return pane;
	}
	
	// makes the board panel and the controls to start and stop the game etc
	Container makePlayPanel() {
		JPanel pane = new JPanel();		
		//pane.setLayout(new BoxLayout(pane, BoxLayout.X_AXIS));

		// make drawable area
		pane.add(makeBoardPanel());
		
		// add buttons
		pane.add(makeButtonPane());
		

		//pane.setBackground(new Color(255,255,204));
		pane.setBorder(BorderFactory.createMatteBorder(1,1,2,2,Color.black));
				
		return pane;
	}

	Container makeTrainPanel() {
		JPanel trainPane = new JPanel();
		//trainPane.setLayout(new BoxLayout(trainPane, BoxLayout.X_AXIS));
		
		trainPane.add(makeSettingPanel());
		trainPane.add(makeParamPanel());

		return trainPane;
	}
	
	// a,g,e parameters for reinforcement learner
	Container makeParamPanel() {
		JPanel parampane = new JPanel();
		parampane.setLayout(new BorderLayout(1,2));
		
		
		JPanel labelpane = new JPanel();
		labelpane.setLayout(new GridLayout(0,1));
		labelpane.add(new JLabel("Death Penalty:", JLabel.RIGHT));
		labelpane.add(new JLabel("Cheese Reward:", JLabel.RIGHT));
		labelpane.add(new JLabel("Alpha:", JLabel.RIGHT));
		labelpane.add(new JLabel("Gamma:", JLabel.RIGHT));
		labelpane.add(new JLabel("Epsilon:", JLabel.RIGHT));
		labelpane.add(new JLabel("Action Selection Method:", JLabel.RIGHT));
		labelpane.add(new JLabel("Learning Method:", JLabel.RIGHT));
		labelpane.add(new JLabel("Epochs to train for:",JLabel.RIGHT));
		labelpane.add(new JLabel("Progress:",JLabel.RIGHT));
		labelpane.add(new JLabel("Epochs done:",JLabel.RIGHT));

		JPanel controlspane = new JPanel();
		controlspane.setLayout(new GridLayout(0,1));
		penalty = new JTextField(20);
		reward = new JTextField(20);
		alpha = new JTextField(20);
		gamma = new JTextField(20);
		epsilon = new JTextField(20);
		controlspane.add(penalty);
		controlspane.add(reward);
		controlspane.add(alpha);
		controlspane.add(gamma);
		controlspane.add(epsilon);

		JPanel actionButtons = new JPanel();
		actionButtons.setLayout(new GridLayout(1,0));
		softmax = new JRadioButton("Softmax");
		greedy = new JRadioButton("Greedy",true);
		ButtonGroup actionButts = new ButtonGroup();
		actionButts.add(softmax);
		actionButts.add(greedy);
		actionButtons.add(softmax);
		actionButtons.add(greedy);
		
		JPanel learnButtons = new JPanel();
		learnButtons.setLayout(new GridLayout(1,0));
		sarsa = new JRadioButton("SARSA");
		qlearn = new JRadioButton("Q-Learning",true);
		ButtonGroup learnButts = new ButtonGroup();
		learnButts.add(sarsa);
		learnButts.add(qlearn);
		learnButtons.add(sarsa);
		learnButtons.add(qlearn);
		epochs = new JTextField(Integer.toString(DEF_EPOCHS));
		progress = new JProgressBar();
		learnEpochsDone = new JLabel("0",JLabel.LEFT);
		 
		controlspane.add(actionButtons);
		controlspane.add(learnButtons);
		controlspane.add(epochs);
		controlspane.add(progress);
		controlspane.add(learnEpochsDone);

		parampane.add(labelpane, BorderLayout.CENTER);
		parampane.add(controlspane, BorderLayout.EAST);
		
		parampane.setBorder(BorderFactory.createTitledBorder("Parameters"));
		
		
		return parampane;
	}
	
	// number of epochs, other settings?, instructions?
	Container makeSettingPanel() {
		JPanel setPane = new JPanel();
		setPane.setLayout(new GridLayout(0,1));
		setPane.add(new JLabel(SETTINGS_TEXT));
		setPane.add(new JLabel(SETTINGS_TEXT2));

		JPanel controls = new JPanel();
		//controls.setLayout(new BoxLayout(controls, BoxLayout.X_AXIS));
		
		startTraining = new JButton("Begin Training");
		startTraining.addActionListener(new ActionListener() {
			public void actionPerformed(ActionEvent e) {
				doTraining();
			}
		});
		stopTraining = new JButton("Stop");
		stopTraining.setEnabled(false);
		stopTraining.addActionListener(new ActionListener() {
			public void actionPerformed(ActionEvent e) {
				endTraining();
			}
		});
		JButton clearPolicy = new JButton("Undo Training");
		clearPolicy.addActionListener(new ActionListener() {
			public void actionPerformed(ActionEvent e) {
				game.setPolicy(rlc.resetLearner());
			}
		});
		controls.add(startTraining);
		controls.add(stopTraining);
		controls.add(clearPolicy);
		
		setPane.add(controls);
		return setPane;
	}

	Container makeBoardPanel() {
		JPanel boardPane = new JPanel();
		boardPane.setLayout(new BoxLayout(boardPane, BoxLayout.Y_AXIS));
		
		bp = new boardPanel(back, BW, BH);
		
		boardPane.add(bp);
		
		// speed control
		speed = new JSlider(0,10,5);
		speed.addChangeListener(new ChangeListener(){
			public void stateChanged(ChangeEvent e) {
				double ratio = 1 - ((double)speed.getValue())/10;
				game.delay = (long)(ratio*DELAY*2);
			}
		});
		speed.setMajorTickSpacing(2);
        speed.setPaintTicks(true);

        //Create the label table.
        Hashtable labelTable = new Hashtable();
        labelTable.put(new Integer( 0 ), new JLabel("Slow") );
        labelTable.put(new Integer( 30 ), new JLabel("Fast") );
        speed.setLabelTable(labelTable);
        speed.setPaintLabels(true);
		boardPane.add(speed);

        //boardPane.setBorder(BorderFactory.createTitledBorder("Game"));

		return boardPane;
	}
	
	Container makeButtonPane() {
		JPanel buttpane = new JPanel();
		buttpane.setLayout(new BoxLayout(buttpane, BoxLayout.Y_AXIS));
		
		// graph of scores
		buttpane.add(chartPane());
		
		// scores
		buttpane.add(scorePanel());
		
		// buttons
		buttpane.add(playControlPanel());
		
		return buttpane;
	}

	Container playControlPanel() {
		JPanel playPanel = new JPanel();
		playPanel.setLayout(new GridLayout(0,2));
		
		startbutt = new JButton("Start");
		startbutt.setActionCommand(START);
		startbutt.addActionListener(this);
		
		JCheckBox continuous = new JCheckBox("Continuous", true);
		continuous.setActionCommand(CONT_CHECK);
		continuous.addItemListener(new ItemListener() {
			public void itemStateChanged(ItemEvent e) {
				if (e.getStateChange() == ItemEvent.DESELECTED)
					game.single=true;
				else game.single = false;
			}
		});

		// add controls to select greedy or rl mouse
		//JPanel learnButtons = new JPanel();
		//learnButtons.setLayout(new GridLayout(1,0));
		JRadioButton greedy = new JRadioButton("Greedy Mouse");
		JRadioButton smart = new JRadioButton("Smart Mouse",true);
		greedy.addActionListener(new ActionListener() {
			public void actionPerformed(ActionEvent e) {
				// set game to use greedy mouse
				game.mousetype = game.GREEDY;
			}
		});
		smart.addActionListener(new ActionListener() {
			public void actionPerformed(ActionEvent e) {
				// set game to use smart mouse
				game.mousetype = game.SMART;
			}
		});
		ButtonGroup mouseButts = new ButtonGroup();
		mouseButts.add(greedy);
		mouseButts.add(smart);

		// add to grid (l-r t-b)
		playPanel.add(startbutt);
		playPanel.add(smart);
		playPanel.add(continuous);
		playPanel.add(greedy);

		playPanel.setBorder(BorderFactory.createTitledBorder("Game Controls"));
		return playPanel;
	}
	
	Container scorePanel() {
		JPanel scorePane = new JPanel();
		//scorePane.setLayout(new BoxLayout(scorePane, BoxLayout.Y_AXIS));
		scorePane.setLayout(new GridLayout(0,2));
		
		// score labels
		ImageIcon cat = new ImageIcon(catImg);
		ImageIcon mouse = new ImageIcon(mouseImg);
		mousescorelabel = new JLabel(MS_TEXT, mouse, JLabel.RIGHT);
		catscorelabel = new JLabel(CS_TEXT, cat, JLabel.RIGHT);

		// reset scores
		//JPanel hbox = new JPanel();
		//hbox.setLayout(new BoxLayout(hbox, BoxLayout.X_AXIS));
		JButton reset = new JButton("Reset Scores");
		reset.addActionListener(new ActionListener(){
			public void actionPerformed(ActionEvent e) {
				mousescore = 0;
				catscore = 0;
				updateBoard();
			}			
		});
		winPerc = new JLabel("", JLabel.RIGHT); // winning percentage label
		//hbox.add(reset);
		//hbox.add(winPerc);
		
		//scorePane.add(hbox);

		scorePane.add(mousescorelabel);
		scorePane.add(winPerc);
		scorePane.add(catscorelabel);
		scorePane.add(reset);
		
		scorePane.setBorder(BorderFactory.createTitledBorder("Scores"));
		return scorePane;
	}
	
	Container chartPane() {
		JPanel ch=new JPanel();
		ch.setLayout(new BorderLayout());
		
		graphPanel = new chartPanel(this);
		graphPanel.setBorder(BorderFactory.createLineBorder(Color.black));
		
		// smoothing control
		smoothSlider = new JSlider(JSlider.HORIZONTAL, 0,99,50);
		smoothSlider.addChangeListener(new ChangeListener(){
			public void stateChanged(ChangeEvent e) {
				double ratio = ((double)smoothSlider.getValue())/100;
				graphPanel.setSmoothing(ratio);
				graphPanel.repaint();
			}
		});
		smoothSlider.setMajorTickSpacing(20);
        smoothSlider.setPaintTicks(true);

        //Create the label table.
        Hashtable labelTable = new Hashtable();
        labelTable.put(new Integer( 0 ), new JLabel("Coarse") );
        labelTable.put(new Integer( 99 ), new JLabel("Smooth") );
        smoothSlider.setLabelTable(labelTable);
        smoothSlider.setPaintLabels(true);

		ch.add(graphPanel, BorderLayout.CENTER);
		ch.add(smoothSlider, BorderLayout.SOUTH);
		
		//ch.add(scorePanel(), BorderLayout.SOUTH);
		
		ch.setBorder(BorderFactory.createTitledBorder("Performance"));
		
		return ch;
	}
	
	/********** Methods to construct panels *************/

	/********** Action handling methods ****************/
	public void actionPerformed(ActionEvent e) {
		if (e.getActionCommand().equals(START)) {
			game.gameOn = true;
		} else if (e.getActionCommand().equals("Draw")) {
			System.out.println("draw test");
			updateBoard();
		}
	}
	/********** Action handling methods ****************/
}

class chartPanel extends JPanel {
	Vector history;
	SwingApplet a;
	final int POINTW=1, POINTH=1, PREFX = 200, PREFY = 100;
	double smoothing = 0.5;
	
	final Color bg=Color.white, fg=Color.blue;
	int MAXSIZE;
	int lastm=0, lastc=0;
	
	
	public chartPanel(SwingApplet a) {
		this.a = a;
		history = new Vector();
	}
	
	public void updateScores() {
		int m = a.mousescore, c = a.catscore;
		int dm = m-lastm, dc = c-lastc;
		lastm=m; lastc=c;
		double score;
		if ((m+c)==0) score = 0;
		else score = ((double)dm) / (dm+dc);
		addScore(score);	
	}
	
	public void paintComponent(Graphics g) {
		MAXSIZE=getWidth()*2;
		
		// draw panel
		g.setColor(bg);
		g.fillRect(0,0,getWidth(),getHeight());
		g.setColor(fg);
		
		double previous=0, thisval, newval;
		for (int x=0; x<history.size(); x++) {
			// draw this point
			
			// smooth with previous values
			thisval = 1 - ((Double)history.elementAt(x)).doubleValue();
			//if (x != startpoint)
			newval = smoothing * previous + (1 - smoothing) * thisval;
			if ((newval >= 0) && (newval <= 1)) previous = newval;
			else System.err.println("Invalid new value: "+newval);
			int yval = (int) (newval * getHeight());
			int xval = x-(history.size() - getWidth());
			//System.out.println("index="+x+" thisval="+thisval+"newval="+newval+" xval="+xval+" yval="+yval+" previous="+previous);
			g.drawOval(xval,yval,POINTW,POINTH);
		}
		
	}
	
	public Dimension getPreferredSize() {
		return new Dimension(PREFX, PREFY);
	}
	
	public void setSmoothing(double s) { smoothing = s; }
	
	void addScore(double s) {
		if (!((s >= 0) && (s<=1))) {
			System.err.println("Graph: rejecting value"+s);
			return;
		}
		
		history.addElement(new Double(s));

		// prune if list too big
		if (history.size() >= MAXSIZE) history.remove(0);

		//System.out.println("Size:"+history.size()+" maxsize:"+MAXSIZE);

		/*
			System.out.println("History being pruned."+Thread.currentThread().getName());
			Vector nVec = new Vector();
			for (int i=(MAXSIZE/3); i<history.size(); i++) {
				nVec.addElement(history.elementAt(i));
			}
			history = nVec;
			System.out.println("Pruning Finished.");
		}*/
	}
}