pia.java.txt

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;


public class pia {
	
	public static void main(String[] args)
	{
		Value b;
		ValueActionFunction vaf;
		float [] probability;
		float [][] policy;
		float [][] oldPolicy;
		float deltaMax =0;
		int policyIteration = 0;//count
		float delta = (float)0.0001;
		float compare =0;//compares old and new policy which is then compared to delta
		boolean policyStable= true;
		int polIter = 0;	 
		int possibleActions = 0; //M-x+1
		float[] stateValue;
		float[][] actionValue;
		float[] prevstateValue;
   		float maxAction;
		int actionChoiceMaxVa;
		int choice; //indexing when assigning new policy
						

		try{
	/*FILE READ---CHECK/CHANGE/CAREFUL WITH FILE PATH...NEXT LINE ONLY NEEDS ATTENTION*/
	  		BufferedReader file = new BufferedReader(new FileReader("C:\\Users\\annad\\Downloads\\School\\CPS841\\ass2\\file1.txt"));
			String line = file.readLine();
			String [] data = line.split(" ");
				
			int M = Integer.parseInt(data[0]);
			
			probability = new float[M+1];//prob(nextState|state and action)
			actionValue = new float[M+1][M+1];//
			stateValue = new float[M+1];
			prevstateValue = new float[M+1];
			policy = new float[M+1][M+1]; //float[action][state]
			oldPolicy = new float[M+1][M+1];
			
			for(int i = 0; i <= M;i++ )
			{	
				probability[i] = Float.parseFloat(data[i+1]);
			}
				
			int k = Integer.parseInt(data[M+2]); 
			float c = Float.parseFloat(data[M+3]);
			float r = Float.parseFloat(data[M+4]);
			float h = Float.parseFloat(data[M+5]);
			float z = Float.parseFloat(data[M+6]);
			
			file.close();

		//Initialize class helpers for Calculating StateValueFunction and ActionValueFunction		
		//StateValueFunction= Value = variable b
		//Value(Iteration,M,CostperUnit,RetailSalesPrice,LostSalesPenalty,HoldingCost,DiscountFactorGamma
		//ValueActionFunction = ValueActionFunction = variable vaf
		//ValueActionFunction(M,CostperUnit,RetailSalesPrice,LostSalesPenalty,HoldingCost,DiscountFactorGamma)
			b = new Value(2,M,c,r,z,h,(float)0.95);
			vaf = new ValueActionFunction(M,c,r,z,h,(float)0.95);	
			
		//set up default policy-every action is equally probable
			for(int x =0;x<=M;x++){
				possibleActions = M-x+1;
				for(int a =0;a<possibleActions;a++){
					policy[a][x] = (float)1/possibleActions;
				}
			}
						
				
			long startTime = System.currentTimeMillis();
			do{
				do{
			    //step zero: save previous StateValues-Default 0 on first round
					for (int x = 0;x<=M;x++){
						if(policyIteration != 0 )
							prevstateValue[x] = stateValue[x];
						else 
							prevstateValue[x]=0;
					}
			
		           //step one : calculate value for every state.
					for(int x =0;x<=M;x++){
						//b.values{currentState,alwaysONE,Currentpolicy,p(s'|s,a)}
						stateValue[x] = b.values(x,1,policy,probability);		
					}

			   //step two: Compare PreviousStateValue with CurrentStateValue					
					deltaMax=0;
					for(int x =0;x<=M;x++){
						compare = Math.abs(stateValue[x]-prevstateValue[x]);
						if(compare>deltaMax){deltaMax=compare;}
					}
					policyIteration++;
				}while(deltaMax > delta);
										
			//PolicyImprovement
			//Compare MaxActionValue with policy action					
				maxAction = 0;
				actionChoiceMaxVa  = 0;
				choice = 0; 	
							
			   //step three : calculate ActionStateValue for every state. 
				//Using ValueStateFunction previously determined
				for(int x =0;x<=M;x++){
					possibleActions = M-x+1;

					for(int a =0;a<possibleActions;a++){
						actionValue[a][x] = vaf.actionvalues(x+a,stateValue, probability);
					}
				}			
							
				for(int x =0;x<=M;x++){
					possibleActions = M-x+1;
					maxAction = 0;
			   //Step four: Determine max ActionStateValue	
					for(int a =0;a<possibleActions;a++)
					{					   
						if(maxAction<actionValue[a][x]){
							maxAction = actionValue[a][x];
							actionChoiceMaxVa = a;}
					}

  		   	   //Step five: Compare policy with max action values
					policyStable = true;
				
					for(int a =0;a<possibleActions;a++){
					
						if( a == actionChoiceMaxVa) choice = 1;
						else choice = 0;

						oldPolicy[a][x] =policy[a][x];
						policy[a][x] = choice;  
						if(oldPolicy[a][x] != policy[a][x]){policyStable = false;}		
					}
				}
				polIter++;
			}while(!policyStable);

			long endTime = System.currentTimeMillis();


							
			for(int x =0;x<=M;x++){
				possibleActions = M-x+1;
				for(int a =0;a < possibleActions;a++){	
					if(policy[a][x]==1)
						System.out.println("Policy for state: " + x + " action: " + a + " is = " + policy[a][x] );
				}
			}
			System.out.println("Iteration through policy convergence = " +policyIteration);
			System.out.println("Iteration through ActionValue convergence = " +polIter);
			System.out.println("That took " + (endTime - startTime) + " milliseconds");
		}catch(IOException e){}
				
	}//end main
}