1: \begin{definition}

2: \label{intermediate value}

3: Let $J_\beta^n\left(x,q_1,l_2\right)$ denote the optimal expected total discounted delay over the next $n$ time slots with initial state $\left(x,q_1,l_2\right)$. Then, we define an intermediate value $T_\beta^n\left(x,q_1,l_2\right)$ as:\emph{

4: \begin{align}

5: & T_\beta^n\left(x,q_1,l_2\right)=\notag\\

6: &\left\{

7: \begin{array}{l l}

8: 	%&T_\beta^n(x,q_1,1)=J_\beta^n(x,q_1,1), \,\,\,\,T_\beta^n(0,0,0)=

9: 	J_\beta^n\left(x,q_1,l_2\right) &\text{\emph{if}}\,\,\mathbf{q}=\mathbf{0}\,\,\text{\emph{or}}\,\, l_2=1\\ 

10: 	%&T_\beta^n(x,q_1,0)=

11: 	\min\{J_\beta^n\left(x,q_1,0\right),J_\beta^n\left(x-1,q_1,1\right)\}&\text{\emph{if}}\,\,x\geq 1, l_2=0\\

12: 	%T_\beta^n(0,q_1,0)=

13: 	\min\{J_\beta^n\left(0,q_1,0\right),J_\beta^n\left(0,q_1-1,1\right)\}&\text{\emph{otherwise}}

14: 	%\,\,q_1\geq 1,\, x=l_2=0.\notag

15: 	%&T_\beta^n(0,0,0)=J_\beta^n(0,0,0).	

16:  \end{array}\right.\notag

17: \end{align}	}

18: %\noindent Note that we omit $J_\beta^n\left(x,q_1-1,1\right)$ for the case $x\geq 1$ and $l_2=0$ in Definiton \ref{intermediate value} due to property (c) in Theorem \ref{thm:optimality}.

19: \noindent As a result, $J_\beta^{n+1}\left(x,q_1,l_2\right)$ is written as:

20: \emph{\begin{align}

21: 	 &J_\beta^{n+1}\left(x,q_1,l_2\right)\notag

22: 	 =  \left(x+q_1+l_2\right)+

23: 	 \beta\lambda T^n_\beta\left(x+1,q_1,l_2\right)\notag\\

24: 	  %\displaybreak

25: 	 &+\beta\mu_\text{mm} p_\text{a} T^n_\beta\left(x,\left(q_1-1\right)^+,l_2\right)+\beta\mu_\text{sub-6} T^n_\beta\left(x,q_1,0\right)\notag\\

26: 	 	 &+\beta\mu_\text{p} T^n_\beta\left(\left(x-1\right)^+,x+q_1-\left(x-1\right)^+,l_2\right).

27: 	 	 	\label{valueiteration}

28: \end{align}}

29: Moreover, $J_\beta^{0}\left(x,q_1,l_2\right)=x+q_1+l_2$.

30: \end{definition}

31: