1: \begin{proof}

2: % Conditioning on $x^l$ and taking expectation with respect to the weights in the $(l+1)$-layer,  

3: % \begin{align}\label{eq_cov}

4: % q^{l+1}_{\alpha, \alpha'}= \mathbb E\,\left[ h^{l+1}_j(\alpha)h^{l+1}_j(\alpha')\middle|x^l \right] 

5: % = \frac{\sigma^2_\omega}{2k+1}\sum_{\beta=-k}^k \frac 1 {c}\sum_{i=1}^c (x^{l}_i(\alpha+\beta)x^{l}_i(\alpha'+\beta)) + \sigma_b^2.

6: % \end{align}

7: % We used the fact that 

8: % \begin{align*}

9: % \mathbb E\, \left[\omega_{ij}^{l+1}(\beta) \omega_{i'j'}^{l+1}(\beta')\right] = 

10: % \begin{cases}

11: % \frac{\sigma_\omega^2}{c(2k+1)}, \quad &{\rm if }\quad (i, j, \beta) = (i', j', \beta')

12: % \\

13: % \\

14: % 0, \quad &otherwise. 

15: % \end{cases}

16: % \end{align*} 

17: % For any fixed pair $(\alpha, \alpha')$, $\{x_i(\alpha)x_i(\alpha')\}_{i}$ are i.i.d. (but not Gaussian), the law of large number gives 

18: % \begin{align}\label{eq_lim}

19: % \lim_{c\to \infty}\frac 1 {c}\sum_{i=1}^c (x^{l}_i(\alpha)x^{l}_i(\alpha'))  = 

20: % \mathbb E\,\left[x^{l}_i(\alpha)x^{l}_i(\alpha') \right] 

21: % \end{align}

22: % Let 

23: % \begin{align*}

24: % \mathcal C(q^l_{\alpha, \alpha'}) = \sigma_\omega^2 \mathbb E\, \left[\phi(z_1)\phi(z_2)\right] + \sigma_b^2 

25: % \end{align*}

26: % with $(z_1, z_2)^T$ drawn from a mean zero Gaussian with covariance matrix 

27: % $[ [q^l_{\alpha,\alpha}, q^l_{\alpha,\alpha'}], [q^l_{\alpha,\alpha'}, q^l_{\alpha',\alpha'}]]$\footnote{Rigorously, ${\mathcal C}$ is a function of $q^l_{\alpha,\alpha}, q^l_{\alpha,\alpha'}$ and $ q^l_{\alpha',\alpha'}$.}. 

28: % Applying \eqref{eq_lim} to \eqref{eq_cov} and letting $c\to\infty$ give 

29: % \begin{align*}

30: %     q^{l+1}_{\alpha, \alpha'} = \frac 1 {2k+1 }\sum_{\beta=-k}^k \mathcal C(q_{\alpha+\beta, \alpha'+\beta}) = \mathcal A \circ \mathcal C(q_{\alpha, \alpha'}).

31: % \end{align*}

32: % \end{proof}