qlsimulator.java

Reinforcement Learning

		if (evt.getActionCommand().equals("LoadMaze")) {
			Utility.show("Loading maze");
			//fileChooser
			fc = new JFileChooser("./mazes/");
			
			int returnVal = fc.showOpenDialog(QLSimulator.this);
			if (returnVal == JFileChooser.APPROVE_OPTION) {
				try {
					File file = fc.getSelectedFile();
					FileInputStream fis = new FileInputStream(file);
					GZIPInputStream gzis = new GZIPInputStream(fis);
					ObjectInputStream in = new ObjectInputStream(gzis);
					myMaze = (Maze)in.readObject();
					in.close();
					mazeStatus = " [ "+myMaze.width+" X "+myMaze.height+" Maze, Wall Penalty:"+((Wall)myMaze.walls.get(0)).penalty +"] | ";
					algorithmStatus = " Q Learning --> Click Initialize";
					
					sqSize = (int)Math.min(Math.floor((this.getSize().width-X-10)/myMaze.width),
					 		  Math.floor((this.getSize().height-Y-10)/myMaze.height));
					
					if(sqSize>200)
                        sqSize=200;
					
					sqSizeTextField.setText(""+sqSize);
					
				} catch(Exception e) {
					Utility.show(e.getMessage());
				}
				ql=null;
			} 
			repaint();
		}
		else if(evt.getActionCommand().equals("Initialize") && myMaze!=null) {
			Utility.show("Q Learning");
			
			double pjog = Double.parseDouble(pjogTextField.getText());
			double learningRate = Double.parseDouble((jLearnRateTextField.getText()));
			double epsilon =Double.parseDouble((jEpsilonTextField.getText()));
			boolean decayingLR = jDecayLRCheckBox.isSelected();
			ql = new QLearning(myMaze, pjog, learningRate, epsilon, decayingLR);
			
			algorithmStatus = " Q Learning ";
			repaint();
		}
		
		else if (evt.getActionCommand().equals("Update") && ql!=null) {
		    ql.setProperty(QLearning.Properties.PJOG,pjogTextField.getText());
		    ql.setProperty(QLearning.Properties.Epsilon,jEpsilonTextField.getText());
		    ql.setProperty(QLearning.Properties.LearningRate,jLearnRateTextField.getText());
		    if(jDecayLRCheckBox.isSelected())
		        ql.setProperty(QLearning.Properties.DecayingLR,new String("true"));
		    else
		        ql.setProperty(QLearning.Properties.DecayingLR,new String("false"));
		}
		
		else if(evt.getActionCommand().equals("Step")) {
			Utility.show("step");
			if(ql!=null)
				if(ql.step()) {
					jStatusLabel.setText("Goal Reached");
				}
			repaint();
		}
		
		else if (evt.getActionCommand().equals("Episode")) {
			Utility.show("Episode");
			int delay = Integer.parseInt(jDelayTextField.getText());
			if(ql!=null) {
				while(!ql.step()) {
				    if(Animate)
				    {
				        Utility.delay(delay);
				        myUpdate(getGraphics());
				    }
				}
				repaint();
			}
		}
		else if (evt.getActionCommand().equals("Cycles")) {
			Utility.show("Cycles");
			int delay = Integer.parseInt(jDelayTextField.getText());
			int numCycles = Integer.parseInt(jCyclesTextField.getText());
			if(ql!=null) {
			    for(int i=0; i<numCycles; i++)
			    {
					while(!ql.step()) {
					    if(Animate)
					    {
					        Utility.delay(delay);
					        myUpdate(getGraphics());
					    }
					}
			    }
				repaint();
			}
		}
		else if(evt.getActionCommand().equals("Refresh")) {
			Utility.show("Refresh");
			repaint();
		}
		else if(evt.getSource().getClass().getName().equals("javax.swing.JTextField")){
            repaint();
        }
		else if(evt.getSource().getClass().getName().equals("javax.swing.JCheckBox")){
			JCheckBox jcb = (JCheckBox)evt.getSource();
			
			if(jcb.getText().equals("Show Values"))
				ShowValue = jcb.isSelected();
			
			if(jcb.getText().equals("Show Policy"))
				ShowPolicy = jcb.isSelected();
			
			if(jcb.getText().equals("Animate"))
				Animate = jcb.isSelected();
			
			repaint();
		}
	}
	
	public void paint(Graphics g)
	{
	    super.paint(g);
		this.myUpdate(g);
	}
	
	public void myUpdate(Graphics g)
	{
		jSeparator1.setSize(4,jPanel.getBounds().height-2);
		jStatusLabel.setText(mazeStatus+algorithmStatus);
		showGrid(getGraphics());
	}
	
	public int sqSize=80;
	int X= 192;
	int Y= 100;
    Color GoldColor = new Color(200,100,55);
	
	public void showGrid(Graphics g)
	{
		sqSize = Integer.parseInt(sqSizeTextField.getText());

		if(ql!=null) {
			drawValues(g);
			if(ShowPolicy)
				drawPolicy(g);
		}
		if(myMaze!=null) {
			drawMaze(g);
			drawGoal(g);
			drawWalls(g);
		}
		if(ql!=null)
		    drawCurrPosition(g, ql.getCurrState());
	}
	
	void drawMaze(Graphics g)
	{
	//	g.setColor(new Color(200,200,250));
	//	g.fillRect(X,Y,sqSize*myMaze.width,sqSize*myMaze.height);
		g.setColor(Color.black);
		for (int counter = 0 ; counter <= myMaze.width; counter++) {
			g.drawLine(X+sqSize*counter, Y+0, X+sqSize*counter, Y+sqSize*myMaze.height);
		}
		for (int counter = 0 ; counter <= myMaze.height; counter++) {
			g.drawLine(X+0,Y+sqSize*counter,X+sqSize*myMaze.width,Y+sqSize*counter);
		}
	}
	
	void drawGoal(Graphics g)
	{
		Vector goals = myMaze.goals;
		State s;
		for (int i=0;i<goals.size();i++)
		{
			s = (State)goals.get(i);
			GraphicsUtil.fillRect(g, X+s.x*sqSize+1, Y+(myMaze.height-1-s.y)*sqSize+1, sqSize-2, sqSize-2, GoldColor);
		}
	}
	
	void drawWalls(Graphics g)
	{
		GraphicsUtil gr = new GraphicsUtil();
		int aX, aY, bX, bY;	//start and end points of the wall
		
		for (int i=0;i<myMaze.walls.size();i++)
		{
			Wall w = (Wall)myMaze.walls.get(i);
			int nodeX = w.x;
			int nodeY = w.y;
			switch(w.dir)
			{
			case Wall.UP:
				aX = nodeX*sqSize;
				bX = (nodeX+1)*sqSize;
				aY = (myMaze.height - nodeY - 1)*sqSize;
				bY = (myMaze.height - nodeY - 1)*sqSize;
				//Utility.show("up wall ax,ay,bx,by= "+aX+","+aY+","+bX+","+bY);
				GraphicsUtil.drawLine(g,X+aX,Y+aY,X+bX,Y+bY,5);
				break;
			case Wall.DOWN:
				aX = nodeX*sqSize;
				bX = (nodeX+1)*sqSize;
				aY = (myMaze.height - nodeY)*sqSize;
				bY = (myMaze.height - nodeY)*sqSize;
				GraphicsUtil.drawLine(g,X+aX,Y+aY,X+bX,Y+bY,5);
				break;
			case Wall.RIGHT:
				aX = (nodeX+1)*sqSize;
				bX = (nodeX+1)*sqSize;
				aY = (myMaze.height-nodeY-1)*sqSize;
				bY = (myMaze.height-nodeY)*sqSize;
				GraphicsUtil.drawLine(g,X+aX,Y+aY,X+bX,Y+bY,5);
				break;
			case Wall.LEFT:
				aX = (nodeX)*sqSize;
				bX = (nodeX)*sqSize;
				aY = (myMaze.height-nodeY)*sqSize;
				bY = (myMaze.height-nodeY-1)*sqSize;
				//Utility.show("left wall ax,ay,bx,by= "+aX+","+aY+","+bX+","+bY);
				GraphicsUtil.drawLine(g,X+aX,Y+aY,X+bX,Y+bY,5);
				break;
			}
		}
	}
	
	void drawCurrPosition(Graphics g, State s)
	{
		int centreX = sqSize*s.x + sqSize/2; 
		int centreY = sqSize*(myMaze.height-1 - s.y) + sqSize/2;
		int radius = sqSize/5;
		Color c = Color.MAGENTA;
		if(!ql.isBestAct)
		    c = Color.YELLOW;
		    
		GraphicsUtil.fillCircle(g, X+centreX, Y+centreY, radius, c);
		GraphicsUtil.drawCircle(g, X+centreX, Y+centreY, radius, 1, Color.black);
		
		GraphicsUtil.drawCircle(g, X+centreX-radius/3, Y+centreY-radius/3, radius/6, 1, Color.black);
		GraphicsUtil.drawCircle(g, X+centreX-radius/3, Y+centreY-radius/3, 1, 1, Color.black);
		
		GraphicsUtil.drawCircle(g, X+centreX+radius/3, Y+centreY-radius/3, radius/6, 1, Color.black);
		GraphicsUtil.drawCircle(g, X+centreX+radius/3, Y+centreY-radius/3, 1, 1, Color.black);
		
		if(ql.receivedPenalty) {
		    GraphicsUtil.drawArc(g,X+centreX-radius/2,Y+centreY+radius/3,radius,2*radius/3,0,180,Color.black);
			//GraphicsUtil.fillArc(g,X+centreX-radius/2,Y+centreY+radius/3,radius,2*radius/3,0,180,Color.black);
		}
		else {
		    GraphicsUtil.drawArc(g,X+centreX-radius/2,Y+centreY,radius,2*radius/3,0,-180,Color.black);
			//GraphicsUtil.fillArc(g,X+centreX-radius/2,Y+centreY,radius,2*radius/3,0,-180,Color.white);
		}
	}

	void drawPolicy(Graphics g)
	{
		//			System.out.println("drawing policy");
		int[][] p = ql.getPolicy();
		int x1=0,x2=0,y1=0,y2=0;
		g.setColor(Color.WHITE);
		for(int i=0;i<p.length;i++) {
			for (int j=0;j<p[i].length;j++) {
				x1 = (i*sqSize)+(sqSize/2);
				y1 = ((myMaze.height-1-j)*sqSize)+(sqSize/2);
				switch (p[i][j]) {
					case Action.UP:
					    x2 = x1;
					    y2 = ((myMaze.height-1-j))*sqSize;
					    break;
					case Action.LEFT:
						x2 = i* sqSize;
						y2 = y1;
						break;
					case Action.DOWN:
						x2 = x1;
						y2 = ((myMaze.height-1-j)*sqSize)+sqSize;
						break;
					case Action.RIGHT:
						x2 = (i*sqSize) + sqSize;
						y2 = y1;
						break;
					default:
						continue;
				}
				g.drawLine(X+x1,Y+y1,X+x2,Y+y2);
			}
		}
	}

	void drawValues(Graphics g)
	{
	    ValueFunction valuefunc = ql.getValueFunction();
        double[][] values = valuefunc.stateValue;
        int max = 1+(int)Math.ceil(valuefunc.getMax());
        
		double[][][] qsa = ql.getQsa();
		
		for (int xval=0 ; xval <myMaze.width; xval++) {
            for (int y=0; y < myMaze.height; y++) {
                int yval = myMaze.height-1-y;
                if (values[xval][y] >= 0) {
                    int red = 155-Math.min((int)(255.0*(values[xval][y])/max),155);
                    int green = 155-Math.min((int)(255.0*(values[xval][y])/max),155);
                    int b = 255-Math.min((int)(255.0*(values[xval][y])/max),220);
                    
                    g.setColor(new Color(red,green,b));
                }
                else
                    g.setColor(GoldColor);
                
                g.fillRect(X+xval*sqSize+1,Y+yval*sqSize+1,sqSize-1,sqSize-1);
            }
		}
		if(!ShowValue)
		    return;
		
		g.setColor(Color.WHITE);
		for(int i=0;i<qsa.length;i++) {
			for(int j=0;j<qsa[i].length;j++) {
				g.drawString(df.format(qsa[i][j][0]), X+(i*sqSize)+(sqSize/2)-5, Y+((myMaze.height-j)*sqSize)-sqSize+15);
				g.drawString(df.format(qsa[i][j][1]), X+(i*sqSize)+sqSize-20, Y+((myMaze.height-j)*sqSize)-(sqSize/2));
				g.drawString(df.format(qsa[i][j][2]), X+(i*sqSize)+(sqSize/2)-5, Y+((myMaze.height-j)*sqSize)-5);
				g.drawString(df.format(qsa[i][j][3]), X+(i*sqSize)+5, Y+((myMaze.height-j)*sqSize)-(sqSize/2));
			}
		}
		
	}
}