Skip to content
Snippets Groups Projects
Commit 2fddf87e authored by Anmulwar, Sweta V (PG/R - Elec Electronic Eng)'s avatar Anmulwar, Sweta V (PG/R - Elec Electronic Eng)
Browse files

FPS to be set by user

parent a8964e1c
No related branches found
No related tags found
No related merge requests found
No preview for this file type
...@@ -17,18 +17,23 @@ namespace QLearningApp.MachineLearning ...@@ -17,18 +17,23 @@ namespace QLearningApp.MachineLearning
private RLSync _qLearningProblem; private RLSync _qLearningProblem;
public QLearning(double gamma, RLSync qLearningProblem) public QLearning(double gamma, RLSync qLearningProblem, double TargetFPS)
{ {
_qLearningProblem = qLearningProblem; _qLearningProblem = qLearningProblem;
_gamma = gamma; // discount factor _gamma = gamma; // discount factor
_qTable = new double[qLearningProblem.NumberOfStates][]; _qTable = new double[qLearningProblem.NumberOfStates][];
/* Need to convert it in the 3-D array */
for (int i = 0; i < qLearningProblem.NumberOfStates; i++) for (int i = 0; i < qLearningProblem.NumberOfStates; i++)
_qTable[i] = new double[qLearningProblem.NumberOfActions]; _qTable[i] = new double[qLearningProblem.NumberOfActions];
for (int i = 0; i < qLearningProblem.NumberOfStates; i++) for (int i = 0; i < qLearningProblem.NumberOfStates; i++)
{ {
for (int j = 0; j < qLearningProblem.NumberOfActions; j++) for (int j = 0; j < qLearningProblem.NumberOfActions; j++)
_qTable[i][j] = 0; _qTable[i][j] = 0;
} }
qLearningProblem.Target_fps = TargetFPS;
/*I can add PE and APL here*/
Console.WriteLine("In QLearning, qLearningProblem.Target_fps = TargetFPS"+TargetFPS);
} }
public void TrainAgent(int numberOfIterations) public void TrainAgent(int numberOfIterations)
...@@ -40,7 +45,7 @@ namespace QLearningApp.MachineLearning ...@@ -40,7 +45,7 @@ namespace QLearningApp.MachineLearning
} }
} }
public QLearningStats Run(int initialStateIndex) public QLearningStats Run(int initialStateIndex)//returning an object
{ {
if (initialStateIndex < 0 || initialStateIndex > _qLearningProblem.NumberOfStates) throw new ArgumentException($"The initial state can be between [0-{_qLearningProblem.NumberOfStates}", nameof(initialStateIndex)); if (initialStateIndex < 0 || initialStateIndex > _qLearningProblem.NumberOfStates) throw new ArgumentException($"The initial state can be between [0-{_qLearningProblem.NumberOfStates}", nameof(initialStateIndex));
...@@ -58,21 +63,23 @@ namespace QLearningApp.MachineLearning ...@@ -58,21 +63,23 @@ namespace QLearningApp.MachineLearning
int actionIndex = _qTable[stateIndex].ToList().IndexOf(_qTable[stateIndex].Max()); int actionIndex = _qTable[stateIndex].ToList().IndexOf(_qTable[stateIndex].Max());
//state = action; //state = action;
double state = _qLearningProblem.SimulatedFPSValues(actionIndex, 4); double state = _qLearningProblem.SimulatedFPSValues(actionIndex, 4);
double statefps_space = _qLearningProblem.fps_space.OrderBy(x => Math.Abs((long)x - state)).First(); double statefps_space = _qLearningProblem.fps_space.OrderBy(x => Math.Abs((long)x - state)).First();
stateIndex = _qLearningProblem.fps_spaceIndex(statefps_space); stateIndex = _qLearningProblem.fps_spaceIndex(statefps_space);
double gamma_action = _qLearningProblem.gamma_space[actionIndex]; double gamma_action = _qLearningProblem.gamma_space[actionIndex];
Console.WriteLine("Current state: " + prev_state + ", Action: " + gamma_action + ", Next state: " + state); Console.WriteLine("Current FPS state: " + prev_state + ", Action: " + gamma_action + ", Next FPS state: " + state);
prev_state = state; prev_state = state;
actions.Add(gamma_action); actions.Add(gamma_action);
stateFPS.Add(state); stateFPS.Add(state);
/*if (_qLearningProblem.GoalStateIsReached(action)) /* Newly added code to make it finite loop */
if (_qLearningProblem.GoalStateIsReached(state))
{ {
result.EndState = action; result.EndState = state;
//need to write code here to end a timer
break; break;
}*/ }
if (counter == 50) /*if (counter == 50)
{ break; } { break; }*/
} }
result.Actions = actions.ToArray(); result.Actions = actions.ToArray();
return result; return result;
...@@ -81,14 +88,18 @@ namespace QLearningApp.MachineLearning ...@@ -81,14 +88,18 @@ namespace QLearningApp.MachineLearning
private void InitializeEpisode(double initialState) private void InitializeEpisode(double initialState)
{ {
//int currentStateIndex = _qLearningProblem.fps_spaceIndex(initialState); //int currentStateIndex = _qLearningProblem.fps_spaceIndex(initialState);
//Console.WriteLine("Enter targetted FPS value ={0}",targetFPS);
double currentState = initialState;//FPS value double currentState = initialState;//FPS value
int counter = 0; int counter = 0;
while (true) while (true)
{ {
currentState = TakeAction(currentState);//currentState= FPS, return should be FPS value currentState = TakeAction(currentState);//currentState= FPS, return should be FPS value
counter++; counter++;
if (counter == 500) /*if (counter == 500)
break;//will run 500 times break;//will run 500 times*/
if (_qLearningProblem.GoalStateIsReached(currentState))
break;
} }
} }
...@@ -98,14 +109,14 @@ namespace QLearningApp.MachineLearning ...@@ -98,14 +109,14 @@ namespace QLearningApp.MachineLearning
int randomIndexActionGamma = _random.Next(0, _qLearningProblem.NumberOfActions); int randomIndexActionGamma = _random.Next(0, _qLearningProblem.NumberOfActions);
//int randomIndexActionDelta = _random.Next(0, _qLearningProblem.NumberOfActions);//choosing a random action from number of actions defined //int randomIndexActionDelta = _random.Next(0, _qLearningProblem.NumberOfActions);//choosing a random action from number of actions defined
double gamma_action = _qLearningProblem.gamma_space[randomIndexActionGamma];//retrieving gamma value form randomIndexAction double gamma_action = _qLearningProblem.gamma_space[randomIndexActionGamma];//retrieving gamma value form randomIndexAction
// double delta_action = _qLearningProblem.delta_space[randomIndexActionGamma]; // double delta_action = _qLearningProblem.delta_space[randomIndexActionGamma];
double delta_action = 20; //fixed for now, can be changed in the future double delta_action = 20; //fixed for now, can be changed in the future
int delta_actionIndex = 0; int delta_actionIndex = 0;
double rho = 0.4; double rho = 0.4;
double saReward = _qLearningProblem.GetReward(currentState, gamma_action, delta_action); double saReward = _qLearningProblem.GetReward(currentState, gamma_action, delta_action);
// Use rho to have a slowly & smoothly changing FPS value after an action is taken // Use rho to have a slowly & smoothly changing FPS value after an action is taken
double nextState = rho * currentState + (1-rho) * _qLearningProblem.SimulatedFPSValues(randomIndexActionGamma, delta_actionIndex); // FPS for time t+1, delta_actionIndex changes to 4 in the RLSync as it is hardcoded double nextState = rho * currentState + (1 - rho) * _qLearningProblem.SimulatedFPSValues(randomIndexActionGamma, delta_actionIndex); // FPS for time t+1, delta_actionIndex changes to 4 in the RLSync as it is hardcoded
double nextStatefps_space = _qLearningProblem.fps_space.OrderBy(x => Math.Abs((long)x - nextState)).First();//need to test first, looking for the closet value of the FPS double nextStatefps_space = _qLearningProblem.fps_space.OrderBy(x => Math.Abs((long)x - nextState)).First();//need to test first, looking for the closet value of the FPS
int nextStateIndex = _qLearningProblem.fps_spaceIndex(nextStatefps_space); int nextStateIndex = _qLearningProblem.fps_spaceIndex(nextStatefps_space);
double nsReward = _qTable[nextStateIndex].Max(); double nsReward = _qTable[nextStateIndex].Max();
...@@ -122,5 +133,6 @@ namespace QLearningApp.MachineLearning ...@@ -122,5 +133,6 @@ namespace QLearningApp.MachineLearning
return initialState; return initialState;
} }
} }
} }
...@@ -10,17 +10,20 @@ namespace QLearningApp ...@@ -10,17 +10,20 @@ namespace QLearningApp
static void Main() static void Main()
{ {
Console.WriteLine("step 1"); Console.WriteLine("step 1");
var qLearning = new QLearning(0.8, new RLSync()); /* Setting the parameters - To set the target FPS */
double targetFPS = 0;
Console.WriteLine("Enter target FPS value");
targetFPS= Convert.ToDouble(Console.ReadLine());
Console.WriteLine("targetFPS from main="+targetFPS);
var qLearning = new QLearning(0.8, new RLSync(),targetFPS);
Console.WriteLine("Training Agent..."); Console.WriteLine("Training Agent...");
qLearning.TrainAgent(2000); qLearning.TrainAgent(2000);
Console.WriteLine("step 2"); Console.WriteLine("step 2");
Console.WriteLine("Training is Done!"); Console.WriteLine("Training is Done!");
Console.WriteLine("Press any key to continue..."); Console.WriteLine("Press any key to continue...");
Console.WriteLine("step 3"); Console.WriteLine("step 3");
Console.ReadLine(); Console.ReadLine();
do do
{ {
......
...@@ -8,18 +8,18 @@ namespace QLearningApp ...@@ -8,18 +8,18 @@ namespace QLearningApp
{ {
private Random _random = new Random(); private Random _random = new Random();
// Actions // Actions
public double[] gamma_space = { 10, 20, 30, 40, 50, 100 }; // the possible values of gamma public double[] gamma_space = { 10, 20, 30, 40, 50, 100 };//{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30};// the possible values of gamma
// private double[] delta_space = { 10, 20, 30, 40, 50, 100 }; // possible values of delta public double[] delta_space = { 10, 20, 30, 40, 50, 100 }; // possible values of delta- 6 values
public double[] delta_space = { 20 }; //public double[] delta_space = { 20 };
public double alpha = 0.2; //original value public double alpha = 0.2; //original value
public double beta = 0.2;//original value public double beta = 0.2;//original value
// State space // State space
public double[] fps_space = { 10, 12, 15, 17, 20, 22, 25, 27, 30 };//PE, APL 0-30 read it form file public double[] fps_space = { 10, 12, 15, 17, 20, 22, 25, 27, 30 };//PE, APL 0-30 read it form file-9 states
private double[][] rewards = new double[5][]; private double[][] rewards = new double[5][];
public double target_fps = 30; public double Target_fps { get; set;}//made it a proerty
/* /*
target for syn window if it is 100ms, target should be given by user not hard-coded target for syn window if it is 100ms, target should be given by user not hard-coded
ini satse: 20 FPS, 100ms latency ini satse: 20 FPS, 100ms latency
...@@ -30,13 +30,13 @@ namespace QLearningApp ...@@ -30,13 +30,13 @@ namespace QLearningApp
public int NumberOfStates => fps_space.Length; public int NumberOfStates => fps_space.Length;
public int NumberOfActions => gamma_space.Length * delta_space.Length; public int NumberOfActions => gamma_space.Length * delta_space.Length;//6*6=36
public double GetReward(double currentState, double action_gamma, double action_delta) public double GetReward(double currentState, double action_gamma, double action_delta)
{ {
//double reward = 1 / (System.Math.Abs(currentState - target_fps)) - alpha * action_gamma - beta * action_delta; //double reward = 1 / (System.Math.Abs(currentState - target_fps)) - alpha * action_gamma - beta * action_delta;
//double reward = 1 / (1+System.Math.Abs(currentState - target_fps)) - alpha * action_gamma/100 - beta * action_delta/100; // gamma nand delta are normalized to [0, 1]; does not perform well when fps is very low //double reward = 1 / (1+System.Math.Abs(currentState - target_fps)) - alpha * action_gamma/100 - beta * action_delta/100; // gamma nand delta are normalized to [0, 1]; does not perform well when fps is very low
double reward = 1 / (1 + System.Math.Abs(currentState - target_fps)) * (1 - alpha * action_gamma / 100 - beta * action_delta / 100); // gamma nand delta are normalized to [0, 1] double reward = 1 / (1 + System.Math.Abs(currentState - Target_fps)) * (1 - alpha * action_gamma / 100 - beta * action_delta / 100); // gamma nand delta are normalized to [0, 1]
return reward; return reward;
} }
...@@ -64,7 +64,14 @@ namespace QLearningApp ...@@ -64,7 +64,14 @@ namespace QLearningApp
} }
return fps_Index; return fps_Index;
} }
public bool GoalStateIsReached(double CurrentState)
{
// Console.WriteLine("target FPS="+target_fps);
return CurrentState >= Target_fps;
}
} }
......
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment