1: %\documentclass[peerreview]{IEEEtran}
2: \documentclass[journal]{IEEEtran}
3: %\documentclass[onecolumn,12pt,draft]{IEEEtran}
4:
5: % package list
6: \usepackage{amssymb,stmaryrd,amsmath,amsfonts,rotating}%amsfonts,amsthm
7: \usepackage[noadjust]{cite}
8: \usepackage{color}%temporary packages
9: \usepackage[vflt]{floatflt}
10: \usepackage{epic}
11:
12: %---------------------------------------------------------------------------------
13: % Usefull macro:
14: % sept.2004. c.m.
15: %---------------------------------------------------------------------------------
16:
17:
18: %---------------------------------------------------------------------------------
19: % temporary options:
20: % comments and corrections when writing
21: %---------------------------------------------------------------------------------
22: \definecolor{TODO}{rgb}{0.6,0.6,0.6} % TO DO!!!!
23: \newcommand{\todo}[1]{{\color{TODO}TO DO: {#1}}\marginpar{\color{TODO}$\leftarrow$ TO DO!}}
24: \definecolor{TOCHECK}{rgb}{0.8,0.8,0.8} % TO CKECK!!!!
25: \newcommand{\tocheck}[1]{\marginpar{\color{TOCHECK}$\leftarrow$ TO CHECK: {#1}}}
26: %---------------------------------------------------------------------------------
27:
28:
29: %---------------------------------------------------------------------------------
30: % LaTex environment
31: %---------------------------------------------------------------------------------
32: \newtheorem{theorem}{Theorem}
33: \newcommand{\btheo}{\begin{theorem}}
34: \newcommand{\etheo}{\end{theorem}}
35: \newcommand{\bproof}{\begin{proof}}
36: \newcommand{\eproof}{\end{proof}}
37: \newtheorem{definition}{Definition}
38: \newcommand{\bdefi}{\begin{definition}}
39: \newcommand{\edefi}{\end{definition}}
40: \newtheorem{fact}{Fact}
41: \newcommand{\bprop}{\begin{fact}}
42: \newcommand{\eprop}{\end{fact}}
43: \newtheorem{corollary}{Corollary}
44: \newcommand{\bcor}{\begin{corollary}}
45: \newcommand{\ecor}{\end{corollary}}
46: \newtheorem{example}{Example}
47: \newcommand{\bex}{\begin{example}}
48: \newcommand{\eex}{\end{example}}
49: \newtheorem{lemma}{Lemma}
50: \newcommand{\blemma}{\begin{lemma}}
51: \newcommand{\elemma}{\end{lemma}}
52: \newtheorem{remark}{Remark}
53: \newcommand{\bremark}{\begin{remark}}
54: \newcommand{\eremark}{\end{remark}}
55: \newtheorem{conj}{Conjecture}
56: %---------------------------------------------------------------------------------
57:
58:
59: %---------------------------------------------------------------------------------
60: % Mathematics:
61: %---------------------------------------------------------------------------------
62: %---------> international notations
63: \newcommand{\reals}{\ensuremath{\mathbb{R}}}
64: \newcommand{\naturals}{\ensuremath{\mathbb{N}}}
65: %---------> mynotations
66: \newcommand{\expectation}{\ensuremath{\mathbb{E}}}
67: \newcommand{\argmin}{\ensuremath{\text{argmin}}}
68: \newcommand{\defas}{\ensuremath{\stackrel{{\vartriangle}}{=}}}
69: %{\ensuremath{\stackrel{\triangle}{=}}}%{\ensuremath{:=}}%{\colon=}} %\stackrel{\vartriangle}{=}
70: %---------------------------------------------------------------------------------
71:
72:
73: %---------------------------------------------------------------------------------
74: % Coding and Information Theory:
75: %---------------------------------------------------------------------------------
76: %---------> Modulation (transmitted bit)
77: \def\0{{\tt 0}} % Ex.: BPSK modulation => 0 is encoded into +1
78: \def\1{{\tt 1}} % Ex.: BPSK modulation => 1 is encoded into -1
79: \def\?{{\tt *}} % erasure symbol
80: \def\g{{\tt g}} % guess symbol
81: \newcommand{\gp}{\ensuremath{\gamma}} % proba for guessing a bit
82: \newcommand{\chm}[1]{\ensuremath{{{\mu^{{\tt{\epsilon}}}_{#1}}}}} %
83: %\def\chm{{\tt {z} }} % channel message
84: \newcommand{\lrm}[1]{\ensuremath{{{\mu^{{\tt{x}}}_{#1}}}}} % left-to-right message
85: \newcommand{\rlm}[1]{\ensuremath{{{\mu^{{\tt{y}}}_{#1}}}}} % right-to-left message
86: \newcommand{\m}{\ensuremath{{{\mu}}}} % message
87: \def\lra{{\tt {x}}} % left-to-right density
88: \def\rla{{\tt {y}}} % right-to-left density
89: %---------> LDPC and Turbo notations
90: \newcommand{\graph}{{\ensuremath{\tt G}}}
91: %\newcommand{\code}{{\ensuremath{\tt C}}}
92: \newcommand{\ldpc}{{{\ensuremath{\text{LDPC}}}}}
93: \newcommand{\ddp}{dd~pair~} %degree distribution pair %d.d.
94: \newcommand{\maxwell}{M~} %Maxwell
95: %\DeclareMathOperator{\ldpc}{LDPC}
96: %\DeclareMathOperator{\graph}{G}
97: \newcommand{\n}{\ensuremath{n}} % code length
98: \newcommand{\drate}{r} % design rate
99: \newcommand{\rate}{\ensuremath{R}} % design rate
100: \newcommand{\ledge}{\ensuremath{\lambda}} % ledge (edge perspective)
101: \newcommand{\redge}{\ensuremath{\rho}} % redge (edge perspective
102: \newcommand{\lnode}{\ensuremath{\Lambda}} % lnode (node perspective)
103: \newcommand{\rnode}{\ensuremath{\Gamma}} % rnode (node perspective)
104: \newcommand{\ldegree}{{\ensuremath{\ensuremath{\tt l}}}} % left degree (regular codes)
105: \newcommand{\rdegree}{{\ensuremath{\ensuremath{\tt r}}}} % right degree (regular codes)
106: %---------> EXIT quantities, thresholds and channel
107: \newcommand{\ih}{\ensuremath{{\tt{h}}}} % intrinsic (or input or channel) entropy
108: \newcommand{\cp}{\epsilon} % channel parameter
109: \newcommand{\xh}{\ensuremath{h}} % extrinsic (or EXIT) entropy
110: \newcommand{\ML}{\ensuremath{\text{ML}}} % Maximum Likelihood
111: \newcommand{\MAP}{\ensuremath{\text{MAP}}} % Maximum A Posteriori
112: \newcommand{\LMAP}{\ensuremath{\text{LMAP}}} % Maximum A Posteriori
113: \newcommand{\BP}{\ensuremath{\text{BP}}} % Belief Propagation
114: \newcommand{\EBP}{\ensuremath{\text{EBP}}} % Extended Belief Propagation
115: \newcommand{\Sh}{\ensuremath{\text{Sh}}} % Shannon
116: \newcommand{\SC}{\ensuremath{\text{SC}}} % Stability Condition
117: \newcommand{\MLsmall}{\ensuremath{\text{\tiny ML}}} % Maximum Likelihood
118: \newcommand{\MAPsmall}{\ensuremath{\text{\tiny MAP}}} % Maximum A Posteriori
119: \newcommand{\LMAPsmall}{\ensuremath{\text{\tiny LMAP}}} % Maximum A Posteriori
120: \newcommand{\BPsmall}{\ensuremath{\text{\tiny BP}}} % Belief Propagation
121: \newcommand{\EBPsmall}{\ensuremath{\text{\tiny EBP}}} % Extended Belief Propagation
122: \newcommand{\Shsmall}{\ensuremath{\text{\tiny Sh}}} % Shannon
123: \newcommand{\SCsmall}{\ensuremath{\text{\tiny SC}}} % Stability Condition
124: \newcommand{\itersmall}{{\tiny \text{$\ell$}}} %
125:
126: %\newcommand{\iteration}{\ensuremath{\ell}} % iteration index during BP iterative decoding
127:
128: \newcommand{\xl}{\ensuremath{{\tt{x}}}}%\xi}} %decoding parameter, FIND A LETTER FOR IT!!!
129: % = left EXIT entropy
130: \newcommand{\xr}{\ensuremath{{\tt{y}}}}%\xi}} % rexit or right EXIT entropy
131: \newcommand{\lxl}{\ensuremath{\underline{\xl}}} %lower parameter
132: \newcommand{\uxl}{\ensuremath{\overline{\xl}}} %upper parameter
133: %--- non-math-mode macros
134: \newcommand{\exitentropy}{EXIT~} %EXIT entropy
135:
136: \newcommand{\qed}{{\hfill \footnotesize $\blacksquare$}}
137: \renewcommand{\mid}{\,|\,}
138: \newcommand{\tablespace}{\vspace{10pt}}
139:
140: %----------> Andrea's definitions
141:
142: \newcommand{\hb}{\bar{h}}
143: \newcommand{\zh}{\hat{z}}
144: \newcommand{\Tc}{T}
145: \newcommand{\Td}{R}
146: \newcommand{\de}{{\text d}}
147:
148:
149: %---- GEXIT
150: \newcommand{\exit}{\ensuremath{\text{EXIT}}}
151: \newcommand{\kl}{k} %kernel in the L-domainKE
152:
153: \newcommand{\gexit}{\ensuremath{\text{GEXIT}}}
154: \newcommand{\gkl}{\ensuremath{\text{GKL}}}
155:
156: \newcommand{\xhker}[1]{\ensuremath{{\kappa_{#1}}}} %kernel for exit
157: %\newcommand{\gexit}{\ensuremath{{\text{\small GEXIT}}}}
158: \newcommand{\gxh}{\ensuremath{{g}}}
159: \newcommand{\gxhker}[1]{\ensuremath{{\tilde{\kappa}_{#1}}}} %kernel for general exit
160: \newcommand{\gxhdabsker}[1]{\ensuremath{{\tilde{\kappa}_{#1}^{|D|}}}} %kernel for general exit
161: \newcommand{\dens}[1]{\mathsf{#1}}
162: \newcommand{\Ldens}[1]{\dens{#1}}
163: \newcommand{\absdens}[1]{\absd{\dens{#1}}}
164: \newcommand{\absLdens}[1]{\absdens{#1}}
165: \newcommand{\Ddens}[1]{\mathfrak{{#1}}}
166: \newcommand{\BEC}{\ensuremath{\text{BEC}}}
167: \newcommand{\EC}{\ensuremath{\text{EC}}}
168: \newcommand{\BSC}{\ensuremath{\text{BSC}}}
169: \newcommand{\BAWGNC}{\ensuremath{\text{BAWGNC}}}
170: \newcommand{\BMS}{\ensuremath{\text{BMS}}}
171: \newcommand{\BM}{\ensuremath{\text{BM}}}
172: \newcommand{\BSCsmall}{\ensuremath{\text{\tiny BSC}}}
173: \newcommand{\BMSsmall}{\ensuremath{\text{\tiny BMS}}}
174: \newcommand{\BECsmall}{\ensuremath{\text{\tiny BEC}}}
175: \newcommand{\BAWGNCsmall}{\ensuremath{\text{\tiny BAWGNC}}}
176:
177: \newcommand{\absd}[1]{|#1|}
178: \newcommand{\absDdens}[1]{\absd{\Ddens{#1}}}
179: \newcommand{\absDdist}[1]{\absd{\mathfrak{#1}}}
180: \newcommand{\Ldensities}{$L$-densities}
181: \newcommand{\Ddensities}{$D$-densities}
182: \newcommand{\Gdensities}{$G$-densities}
183:
184: % new definitions for GEXIT kernels
185: \newcommand{\exitkl}[1]{l(#1)} % EXIT kernel in L domain
186: \newcommand{\gexitkl}[2]{l^{#1}(#2)} % GEXIT kernel in L domain
187: \newcommand{\gexitkd}[2]{d^{#1}(#2)} % GEXIT kernel in D domain
188: \newcommand{\gexitkabsd}[2]{|d|^{#1}(#2)} % GEXIT kernel in |D| domain
189: \newcommand{\exitf}[1]{h^{#1}} % EXIT function
190: \newcommand{\exitfc}[2]{h^{#1}_{#2}} % EXIT function
191: \newcommand{\exitfi}[1]{h^{#1}_i} % EXIT function
192: \newcommand{\gexitf}[1]{g^{#1}} % GEXIT function
193: \newcommand{\gexitfdual}[1]{\hat{g}^{#1}} % GEXIT function
194: \newcommand{\gexitfc}[2]{g^{#1}_{#2}} % GEXIT function
195: \newcommand{\gexitfi}[1]{g_i^{#1}} % GEXIT function
196: \newcommand{\gexitfroot}[1]{g_{\text{r}}^{#1}} % GEXIT function
197: \newcommand{\gexitfleaf}[1]{g_{\text{l}}^{#1}} % GEXIT function
198:
199: % definitions from ITW paper
200: \newtheorem{thm}{Theorem}
201: \newtheorem{coro}{Corollary}
202: \newcommand{\newcaption}[1]{\caption{\footnotesize{#1}}}
203: %\newtheorem{lemma}{Lemma}
204: %\newtheorem{example}{Example}
205: %\newcommand{\nldpc}[2]{\rm{LDPC}(#1, #2)}
206: %\newcommand{\ldpc}[2]{\rm{LDPC}(#1, #2)}
207: %\newcommand{\expldpc}[3]{\rm{EXPLDPC}(#1, #2, #3)}
208: \newcommand{\eldpc}[3]{{\rm LDPC}(#1, #2, #3)}
209: %\newcommand{\ldpco}[2]{\rm{LDPC}_{\circ}(#1, #2)}
210: %\newcommand{\ldpcd}[2]{\rm{LDPC}_{<}(#1, #2)}
211: %\newcommand{\ldpcp}[2]{\rm{LDPC}_{P}(#1, #2)}
212: %\newcommand{\eldpcp}[3]{\rm{LDPC}_{P}(#1, #2, #3)}
213: %\newcommand{\ledge}{\ensuremath{{\lambda}}}
214: %\newcommand{\ldegree}{\ensuremath{{\tt{l}}}} %d_{\lambda}
215: %\newcommand{\redge}{\ensuremath{{\rho}}}
216: %\newcommand{\rdegree}{\ensuremath{{\tt{r}}}} %d_{\rho}
217: %\newcommand{\lnode}{\ensuremath{{\Lambda}}}
218: %\newcommand{\rnode}{\ensuremath{{P}}}
219: %\newcommand{\exit}{{\sf EXIT}}
220: %\newcommand{\gexit}{{\sf GEXIT}}
221: \newcommand{\mmse}{{\sf MMSE}}
222:
223: \def\cX{{\cal X}}
224: \def\cY{{\cal Y}}
225: \def\uX{\underline{X}}
226: \def\uY{\underline{Y}}
227: \def\0t{{\tt 0}}
228: \def\1t{{\tt 1}}
229: \def\sfa{{\sf a}}
230: \def\sfb{{\sf b}}
231: \def\tk{\tilde{k}}
232: \def\Ct{{\tt C}}
233: \def\snr{{\sf snr}}
234: \def\R{{\mathbb R}}
235: \def\E{{\mathbb E}}
236:
237: \newcommand{\iter}{\ell}
238: \newcommand{\drmax}{{\tt r}_{\text{max}}}
239: \newcommand{\myscriptsize}{\tiny}
240: \newcommand{\entropy}{H} % entropy operator
241: \newcommand{\gentropy}{G} % entropy operator
242: \newcommand{\conv}{\star}
243: \DeclareMathOperator{\perr}{\mathfrak{E}} % probability of error operator
244: \DeclareMathOperator{\batta}{\mathfrak{B}} % Battacharya operator
245: % \newcommand{\ent}{\ensuremath{{\tt{h}}}}
246: \newcommand{\ent}{\ensuremath{{\tt{h}}}}
247: \newcommand{\logtwo}{\log_2}
248: \newcommand{\lognat}{\log}
249:
250:
251: %\hyphenation{op-tical net-works semi-conduc-tor}
252:
253: \begin{document}
254: \title{The Generalized Area Theorem and Some of its Consequences}
255: \author{
256: Cyril~M\'easson,$^\dagger$ %
257: \thanks{$\dagger$ EPFL, School for Computer and Communication Sciences, CH-1015 Lausanne, Switzerland. E-mail: cyril.measson@epfl.ch} %
258: Andrea~Montanari,$^*$%
259: \thanks{$*$ ENS, Laboratoire de Physique Th\'eorique, F-75231 Paris, France. E-mail: montanar@lpt.ens.fr} %
260: Tom Richardson,$^+$%
261: \thanks{$+$ Flarion Technologies, Bedminster, USA E-mail: tjr@flarion.com} %
262: and R\"udiger Urbanke%
263: $^\ddagger$\thanks{$\ddagger$ EPFL, School for Computer and Communication Sciences, CH-1015 Lausanne, Switzerland. E-mail: ruediger.urbanke@epfl.ch} %
264: \thanks{Parts of the material were presented in \cite{MMRU04, MMRU05}.}
265: } %
266:
267: \maketitle
268:
269:
270: \begin{abstract}
271: There is a fundamental relationship between belief propagation and
272: maximum a posteriori decoding. The case of transmission over the
273: binary erasure channel was investigated in detail in a companion paper.
274: This paper investigates the extension to general
275: memoryless channels (paying special attention to the binary case).
276: An area theorem for transmission over general memoryless channels is
277: introduced
278: and some of its many consequences are discussed. We show that
279: this area theorem gives rise to an upper-bound
280: on the maximum a posteriori threshold for sparse graph codes.
281: In situations where this bound is tight, the extrinsic
282: soft bit estimates delivered by the belief propagation decoder coincide with
283: the correct a posteriori probabilities above the maximum a
284: posteriori threshold.
285: More generally, it is conjectured that the fundamental relationship between the
286: maximum a posteriori and the belief propagation decoder which
287: was observed for transmission over the binary erasure channel
288: carries over to the general case. We finally demonstrate
289: that in order for the design rate of an ensemble
290: to approach the capacity under belief propagation decoding the
291: component codes have to be perfectly matched, a statement which
292: is well known for the special case of transmission over the binary
293: erasure channel.
294: \end{abstract}
295:
296: \begin{keywords}
297: belief propagation, maximum a posteriori, maximum likelihood, Maxwell construction,
298: threshold, phase transition, Area Theorem, $\exit$ curve, entropy
299: \end{keywords}
300:
301: %\IEEEpeerreviewmaketitle
302:
303: \section{Introduction}
304: \PARstart{I}{t} was shown in \cite{MeU03,MMU04,MMU05} that, when transmission takes place over the
305: binary erasure channel ($\BEC$) using sparse graph codes, there exists a surprising and fundamental relationship
306: between the belief propagation ($\BP$) and the maximum a posteriori ($\MAP$) decoder.
307: This relationship emerges in the limit
308: of large blocklengths. Operationally, this relationship is furnished
309: for the $\BEC$ by the so-called Maxwell decoder. This decoder bridges the gap between
310: $\BP$ and $\MAP$
311: decoding by augmenting the $\BP$ decoder with an additional ``guessing''
312: device. Analytically, the relationship between $\BP$ and $\MAP$ decoding
313: is given in terms of the so-called extended $\BP$ $\exit$ ($\EBP$ $\exit$) function.
314: Fig.~\ref{fig:multijump} shows this curve (double ``S''-shaped curve)
315: for transmission over the $\BEC$ and
316: the ensemble LDPC($\frac{3x+3x^2+4x^{13}}{10},x^6$) (the degree distributions are
317: from an edge perspective).
318: The $\BP$ $\exit$ curve
319: is the ``envelope'' of the $\EBP$ $\exit$ curve (let a ball run slowly down the slope).
320: The $\MAP$ $\exit$ curve on the other hand
321: is conjecture to be derived in general from the
322: $\EBP$ $\exit$ curve by the so-called Maxwell
323: construction. This Maxwell construction consists of converting the
324: $\EBP$ $\exit$ curve
325: into a single-valued function by ``cutting'' the $\EBP$ $\exit$ curve at the
326: two ``S''-shaped spots in such a way that there is a local balance of the cut areas.
327: \begin{figure}[hbt]
328: \vspace{25bp}
329: \centering
330: \setlength{\unitlength}{0.5bp}
331: \begin{picture}(105,153)
332: \put(-45,0){\includegraphics[scale=0.62]{EXITconclusion}}
333: \put(36,-7){\makebox(0,0){\small{$\ent^{\MAPsmall}$}}}
334: \put(7,-7){\makebox(0,0){\small{$\ent^{\BPsmall}$}}}
335: \put(116,88){\makebox(0,0){\small{$\exitf {} (\ent)$}}}
336: \put(46,105){\makebox(0,0){\small{$\exitf{\BPsmall}(\ent)$}}}
337: \end{picture}
338: \caption{\label{fig:multijump}
339: The $\EBP$ $\exit$ curve (double ``S''-shaped curve), the corresponding
340: $\BP$ $\exit$ curve (dashed and solid line;
341: the ``envelope'' of the $\EBP$ $\exit$ curve)
342: and the $\MAP$ $\exit$ curve (thick solid line;
343: constructed by ``cutting'' $\EBP$ $\exit$ at the
344: two ``S''-shaped spots in such a way that there is a local balance of the
345: areas shown in gray)
346: for the ensemble LDPC($\frac{3x+3x^2+4x^{13}}{10},x^6$). }
347: \vspace{5bp}
348: \end{figure}
349: A detailed discussion of this relationship in the case of transmission
350: over the $\BEC$ can be found in \cite{MMU05}.
351: Let us summarize. For transmission over the $\BEC$ using sparse
352: graph codes from long ensembles,
353: $\BP$ decoding is asymptotically characterized
354: by its $\BP$ $\exit$ curve and $\MAP$ decoding is characterized by its $\MAP$ $\exit$ curve.
355: These two curves are linked via the $\EBP$ $\exit$ curve.
356:
357:
358: \subsection{Overview of Results}
359:
360: The pleasing picture shown in Fig.~\ref{fig:multijump}
361: seems to have a fairly complete analog in the general setting.
362: Unfortunately we are not able to prove this claim in any generality.
363: But we show how several of the key ingredients can be suitably extended
364: to the general case and we will be able to prove some of their fundamental
365: properties.
366:
367: Namely, we introduce a general area theorem (GAT). This
368: area theorem, when applied to the $\BEC$, leads back to the notion of $\exit$ functions
369: as shown in the companion paper \cite{MMU05}.
370: For the general case however, it is necessary to use a distinct
371: function (but similar in many respects to $\exit$).
372: We call it
373: the generalized $\exit$ ($\gexit$) function. We then show that $\gexit$ functions share
374: some of the key properties with $\exit$ functions. In particular, we are able
375: to extend the upper-bound on the $\MAP$ threshold presented in \cite{MeU03} (or, more generally, the lower
376: bound on the conditional entropy) to general channels.
377:
378: In \cite{GSV04c,GSV05} Guo, Shamai and Verd{\'u} showed that for Gaussian channels
379: the derivative
380: (with respect of the signal-to-noise ratio) of the mutual information,
381: is equal to the mean square error (MSE),
382: and in \cite{GSV04c} they showed that a similar
383: relationship holds for Poisson channels.
384: One can think of $\gexit$ functions as providing such
385: a relationship in a more general
386: setting (where the generalization
387: is with respect to the admissible channel families).
388: For some channel families,
389: $\gexit$ functions have particularly nice interpretations.
390: E.g., for Gaussian channels,
391: we not only have the interpretation of the derivative in terms of the MSE
392: detector, but this interpretation can be simplified
393: even further in the binary case: the derivative of the mutual information
394: can be seen as the ``magnetization'' of the system
395: as was shown by
396: Macris in \cite{Mac05}.
397: The results in \cite{GSV05i}, which have appeared since the introduction
398: of GEXIT functions in \cite{MMRU04}, can be reformulated
399: to give an interpretation of GEXIT functions
400: for the class of additive channels (see also \cite{Zak05}).
401: It is likely that interpretations for other classes
402: of channels will be found in the future.
403:
404: \subsection{Paper Outline}
405: In Section \ref{sec:review} we review the necessary background material
406: and in particular recall the GAT first stated in \cite{MMRU04}.
407: Starting from this GAT, we introduce in Section \ref{sec:gexit}
408: $\gexit$ functions. We will see that for transmission over the $\BEC$,
409: $\gexit$ functions coincide with standard $\exit$ functions, but that
410: this is no longer true for general channels.
411: In Section \ref{sec:asymptotic} we then concentrate on
412: LDPC ensembles. In particular we define the quantities which
413: appear in the asymptotic setting.
414: In Section \ref{sec:basicproperties} we
415: then prove one of the fundamental properties of $\gexit$ functions, namely
416: that $\gexit$ kernels preserve the ordering implied by physical degradation.
417: This fact is then exploited in Section \ref{sec:upperbound}, where we show
418: how to compute an upper bound on the threshold under $\MAP$ decoding
419: (or, more generally, a lower bound on the conditional entropy) by considering
420: the \BP\ $\gexit$ function, which results
421: from the regular $\gexit$ function if we substitute
422: the MAP density by its equivalent BP density.
423: In Section \ref{sec:egexit} we define extended BP $\gexit$ (EBP $\gexit$) functions which
424: include the unstable branches, present several examples of these function
425: and discuss how they provide a bridge between belief propagation and
426: maximum a posteriori decoding.
427: Several properties of EPB $\gexit$ functions are discussed
428: in Section \ref{sec:HowToEBP} together with a numerical procedure for
429: constructing them. We show that they satisfy an area theorem as well.
430: Section \ref{sec:Regularity} presents some partial results on the smoothness
431: and uniqueness of EBP $\gexit$ functions.
432: In Section \ref{sec:mapversusbp} we show the surprising fact that, in case the
433: previously computed upper bound on the $\MAP$ threshold is tight, then the
434: a posteriori probabilities on the bits are equal to the corresponding
435: $\BP$ estimates. Section \ref{sec:Matching} contains a proof
436: that iterative coding systems cannot achieve reliable communication
437: above capacity, using only density evolution and the area theorem
438: (and not the standard Fano inequality).
439: A matching condition for component codes of capacity achieving
440: sequences follows.
441: In the appendices we collect some technical derivations and
442: a discussion of several equivalent forms of the $\gexit$
443: functions for Gaussian channels.
444: We finally conclude
445: with some remarks in Section \ref{sec:conclusion}.
446: %
447: %********************************************************************
448: %
449: \section{Review and Notations}
450: \label{sec:review}
451: Let ${\cal{X}}$ denote the channel input alphabet
452: (which we always assume finite)
453: and ${\cal{Y}}$ the channel output alphabet (typically, ${\cal{Y}}={\mathbb{R}}$).
454: All channels considered in this paper are {\em memoryless} (M).
455: Rather than looking at a single memoryless channel, we usually consider
456: {\em families} of memoryless channels parameterized by a real-valued
457: parameter $\cp$, which we denote by $\{\text{M}(\cp)\}_{\cp}$.
458: Each channel from such
459: a family is characterized by its transition probability density
460: $p_{Y \mid X}(y \mid x)$ (where $x\in\cX$ and $y\in\cY$).
461: We adopt here the convention of formally denoting channels by
462: their transition density even when such a density does not exist,
463: and write $\int f(y) p_{Y \mid X}(y \mid x)\text{d}y$ as a proxy for
464: the corresponding expectation.
465:
466: Transmission over {\em binary}-input
467: {\em memoryless} output-{\em symmetric}\footnote{A binary
468: memoryless channels is said to be symmetric
469: (or, more precisely, output-symmetric) when
470: the transition probability verifies
471: $p_{Y|X}(y|+1)=p_{Y|X}(y|-1)$.} ($\BMS$) channel plays
472: a particularly important role.
473: In this case, it will be convenient to
474: assume that the input bit $X_i$ takes values
475: $x_i\in {\cal{X}}\defas\{+1,-1\}$.
476: The channel indexed by parameter $\cp$ is
477: generically denoted by $\BMS(\cp)$.
478:
479: In the sequel
480: we will often assume that the {\em channel} family $\{\BMS(\cp)\}_{\cp}$
481: is {\em ordered by physical degradation} (see
482: \cite{RiU05} for a discussion of this concept).
483: It is well known that the standard families
484: $\{\BEC(\cp)\}_{\cp=0}^{1}$ (binary erasure
485: channels with erasure parameter $\epsilon$),
486: $\{ \text{BSC}(\cp)\}_{\cp=0}^{\frac{1}{2}}$
487: (binary symmetric channels with cross-over probability $\epsilon$), and
488: $\{ \text{BAWGNC}(\sigma)\}_{\sigma=0}^{\infty}$
489: (binary-input additive white Gaussian noise channels $Y=X+N$
490: where $X$ takes values in ${\cal{X}}$ and the noise $N$
491: has standard deviation $\sigma$ and zero-mean) all have
492: this property. For notational simplicity we will use a shorthand
493: and say that a channel family is {\em degraded}.
494:
495: In the binary case, an important role is played by the distribution of
496: the log-likelihood ratio
497: $L \defas \log\frac{p_{Y|X}(Y|+1)}{p_{Y|X}(Y|-1)}$, assuming $X=1$.
498: We denote the corresponding density by $\Ldens{c}(l)$ and call it
499: an $L$-density.
500: In fact, without loss of generality we can assume that
501: the log-likelihood ratio ($L$) mapping,
502: $y\mapsto \log\frac{p_{Y|X}(y|+1)}{p_{Y|X}(y|-1)}$,
503: is already included in the channel description.
504: This is justified since the
505: random variable $L$ constitutes a
506: sufficient statistic.
507: This inclusion of the $L$-processing is equivalent to assuming
508: that $p_{Y|X}(y|+1)= \Ldens{c}(l)$.
509: Further facts regarding $\BMS$ channels can be found in \cite{RiU05}.
510: As far as LDPC and iterative coding systems are concerned,
511: we will keep the formalism introduced in the
512: companion paper \cite{MMU05} and which is found, e.g.,
513: in \cite{LMSS01,LMSS01b,RiU01,RSU01}.
514:
515: In the case of a {\em non-binary} input alphabet ${\cal X}$,
516: the log-likelihood mapping will be replaced by the
517: `canonical' representation of the channel output
518: $y\mapsto \nu(y) \defas\{p_{Y|X}(y|x)/z(y)\, :\, x\in{\cal X} \}$,
519: where $z(y) \defas \sum_{x\in {\cal X}} p_{Y|X}(y|x)$.
520: Notice that $\nu(y)$ belongs to the $(|{\cal X}|-1)$-dimensional
521: simplex $S_{|{\cal X}|-1}$.
522: In the binary case, the log-likelihood ratio is just
523: a particular parametrization of the one-dimensional simplex.
524:
525: In what follows we will often be concerned with how certain
526: quantities (e.g., the conditional entropy $H(X \mid Y)$)
527: behave as we change the channel parameter. In order
528: to ensure that the involved objects exits we need to impose
529: some regularity conditions on the channel family with respect
530: to the channel parameter. This can be done in various ways, but
531: to be concrete we will impose the following restriction.
532: \bdefi[Channel Smoothness]
533: Consider a family of memoryless channels with input and output alphabets
534: ${\cal X}$ and ${\cal Y}$, respectively,
535: and characterized by their transition
536: probability $p_{Y|X}(y | x)$ (with $y$ taking the canonical form described
537: above).
538: Assume that the family is parameterized by $\cp$,
539: where $\cp$ takes values in some interval $I \subseteq {\mathbb R}$.
540: The channel family is said to be {\em smooth}
541: with respect to the parameter $\cp$ if for all $x\in {\cal X}$ and
542: all bounded continuously differentiable functions
543: $f(y)$ on $S_{|{\cal X}|-1}$, the integral
544: $\int f(y) p_{Y | X}(y | x) \text{d} y$ exists
545: and is a continuously differentiable function with respect to $\cp$, $\cp \in I$.
546: \edefi
547: In the sequel we often say as a shorthand
548: that a {\em channel} $\BMS(\cp)$ is smooth
549: to mean that we are transmitting over the channel
550: $\BMS(\cp)$ and that the {\em channel family} $\{\BMS(\cp)\}_{\cp}$ is
551: smooth at the point $\cp$. If $\BMS(\cp)$ is smooth, the derivative
552: $\frac{\text{d}\phantom{\cp}}{\text{d}\cp}\int f(y) p_{Y | X}(y | x)
553: \text{d} y$ exists and
554: is a linear functional of $f$. It is therefore consistent to formally
555: {\em define} the derivative of $ p_{Y | X}(y | x)$ with respect
556: to $\cp$ by setting
557: %
558: \begin{align}
559: %
560: \frac{\text{d}\phantom{\cp}}{\text{d}\cp}\int\! f(y) p_{Y | X}(y | x)
561: \, \text{d} y \defas
562: \int\! f(y) \frac{\text{d}p_{Y | X}(y | x)}{\text{d}\cp} \,
563: \text{d} y\, .\label{eq:FormalDerivative}
564: %
565: \end{align}
566: %
567: For a large class of channel families it is straightforward
568: to check that they are smooth.
569: This is e.g. the case if $\{\cal Y\}$ is finite and the
570: transition probabilities are differentiable functions of $\cp$,
571: or if it admits a density with respect to the Lebesgue measure,
572: and the density is differentiable for each $y$.
573: In these cases, the formal derivative (\ref{eq:FormalDerivative})
574: coincides with the ordinary derivative.
575:
576: \bex[Smooth Channels] \label{ex:channeldefinition}
577: It is straightforward to check that the families
578: $\{\BEC(\cp)\}_{\cp=0}^{1}$,
579: $\{ \text{BSC}(\cp)\}_{\cp=0}^{\frac{1}{2}}$, and
580: $\{ \text{BAWGNC}(\sigma)\}_{\sigma=0}^{\infty}$
581: are all smooth.
582: \eex
583:
584: In the case of transmission over a $\BMS$ channel
585: it is useful to parameterize
586: the channels in such a way that the parameter reflects
587: the channel entropy. More precisely, we denote
588: by $\ent$ the conditional entropy $H(X|Y)$ when
589: the channel input $X$ is chosen uniformly at random from $\{+1,-1\}$, and
590: the corresponding output is $Y$. Consider a family of
591: $\BMS$ channels characterized by their $L$-densities.
592: We then write this family of $L$-densities as
593: $\{\Ldens{c}_{\ent}\}_{\ent}$ if
594: $\entropy(\Ldens{c}_{\ent})=\ent$, where the {\em entropy
595: operator} is defined as (see, e.g., \cite{RiU05})
596: \begin{align}
597: \label{equ:entropy}
598: \entropy(\Ldens{c}) & \defas \int_{-\infty}^{\infty}
599: \Ldens{c}(y) \logtwo(1+e^{-y}) \text{d}y =
600: \int_{-\infty}^{\infty}
601: \Ldens{c}(y) \exitkl y \text{d}y.
602: \end{align}
603: This integral always exists as can be seen by writing it
604: in the equivalent form as Rieman-Stieltjes integral
605: $ \int_{0}^{\infty}
606: h_2\left(\frac{e^{-y}}{1+e^{-y}} \right)\text{d}\absLdens{C}(y)$.
607: In the above definition we have introduced the {\em kernel}
608: $\exitkl y \defas \logtwo(1+e^{-y})$. For reasons that
609: will become clearer in Lemma \ref{lem:exitlinear},
610: we call $\exitkl y$ the $\exit$ kernel.
611:
612: The channel family is said to be {\em complete} if
613: $\ent$ ranges from $0$ to $1$. For the binary erasure channel the natural parameter $\cp$
614: (the erasure probability) already
615: represents an entropy. Nevertheless, to be consistent we will write
616: in the future $\BEC(\ent)$. By some abuse of notation,
617: we write BSC$(\ent)$ to denote the BSC with cross-over probability
618: equal to $\cp(\ent)=h_2^{-1}(\ent)$,
619: where $h_2(x) \defas -x \log_2(x)-(1-x) \log_2(1-x)$, the
620: binary entropy function. In the same manner,
621: BAWGNC$(\ent)$ denotes the BAWGNC with a standard deviation of the
622: noise such that the channel entropy is equal to $\ent$.
623:
624: We will encounter cases where it is useful to allow each
625: bit of a codeword to be transmitted through a different (family of)
626: $\BMS$ channel(s). By some abuse of notation, we will denote the
627: $i^{\text{th}}$ channel family by $\{\BMS(\ent_i)\}_{\ent_i}$.
628: %and its corresponding family of $L$-density
629: %by $\Ldens{c}_i$.
630: A situation in which this
631: more general view appears naturally is when we consider
632: punctured ensembles. We can describe this case by assuming that
633: some bits are passed through an erasure channel with erasure
634: probability equal to one, whereas the remaining bits are passed
635: through some other $\BMS$ channel. In such cases it is convenient
636: to assume that all individual families $\{\BMS(\ent_i)\}_{\ent_i}$
637: are parameterized
638: in a smooth (differentiable) way by a single
639: real parameter, call it $\cp$, i.e., $\ent_i=\ent_i(\cp)$.
640: In this way, by changing $\cp$ all channels change according to $\ent_i(\cp)$
641: and they describe a path through ``channel space''.
642:
643: The general area theorem (GAT), first introduced in \cite{MMRU04},
644: plays center stage in the remainder of this paper.
645: \btheo[General Area Theorem]
646: \label{theo:generalareatheorem}
647: Let $X$ be chosen with probability $p_X(x)$ from ${\cal X}^n$.
648: Let the channel from $X$ to $Y$ be memoryless, where
649: $Y_i$ is the result of passing $X_i$ through the smooth
650: family $\{\text{M}(\cp_i)\}_{\cp_i}$, $\cp_i \in I_i$.
651: Let $\Omega$ be a further observation of $X$ so that
652: $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
653: Then
654: \begin{align}
655: \label{eq:generalareatheorem}
656: \text{d} H(X \mid Y,\Omega) & =
657: \sum_{i=1}^\n \frac{\partial H(X_i \mid Y,\Omega ) }{\partial \cp_i} \text{d}\cp_i.
658: \end{align}
659: \etheo
660: \bproof
661: For $i \in [\n]$, the entropy rule gives $H(X \mid Y,\Omega)=H(X_i \mid Y,\Omega)+H(X_{\sim i} \mid X_i,Y,\Omega)$.
662: We claim that
663: \begin{align}
664: \label{equ:factorization}
665: p(X_{\sim i} \mid X_i,Y,\Omega) = p(X_{\sim i} \mid X_i,Y_{\sim i},\Omega),
666: \end{align}
667: which is true since the channel is memoryless and $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
668: Furthermore $H(X_i \mid Y,\Omega)$ is differentiable with
669: respect to $\cp_i$ as a consequence of the channel smoothness
670: (it is straightforward to write the conditional entropy as expectation
671: of a differentiable kernel, cf.~Lemma \ref{lemma:gexitBM} and remarks below).
672: Therefore, $H(X_{\sim i} \mid X_i,Y,\Omega)=H(X_{\sim i} \mid X_i,Y_{\sim i},\Omega)$ and
673: $\frac{\partial H(X \mid Y,\Omega ) }{\partial \cp_i} = \frac{\partial H(X_i \mid Y,\Omega) }{\partial \cp_i}$.
674: From this the total derivate as stated in
675: (\ref{eq:generalareatheorem}) follows
676: immediately.
677: \eproof
678:
679:
680: \section{$\gexit$ Functions}
681: \label{sec:gexit}
682: Let $X$ be chosen with probability $p_X(x)$ from ${\cal X}^n$.
683: Assume that the $i^{\text{th}}$ component of $X$ is transmitted
684: over a memoryless erasure channel (not necessarily binary) with
685: erasure probability $\cp_i$, denote it by $\EC(\cp_i)$.
686: Then
687: $H(X_i \mid Y) = \bar{\cp}_i H(X_i \mid Y_i=X_i, Y_{\sim i}) + \cp_i
688: H(X_{i} \mid Y_i=?, Y_{\sim i}) = \cp_i
689: H(X_{i} \mid Y_{\sim i})$.
690: Apply equation (\ref{eq:generalareatheorem}) in
691: Theorem \ref{theo:generalareatheorem}
692: assuming that $\cp_i=\cp$, $i \in [n]$.
693: To remind ourselves that $Y$ is a function of the
694: parameter $\cp$ we write $Y(\cp)$.
695: Then
696: \begin{align*}
697: \frac{1}{n} \frac{\text{d}\phantom{\cp}}{\text{d} \cp} H(X \mid Y(\cp))
698: & = \frac{1}{n} \sum_{i=1}^{n} H(X_{i} \mid Y_{\sim i}(\cp)).
699: \end{align*}
700: The function $h^{}_i(\cp) \defas H(X_{i} \mid Y_{\sim i}(\cp))$
701: is known in the literature as the $\exit$
702: function associated to the $i^{\text{th}}$ bit
703: of the given code and
704: $h^{}(\cp) \defas \frac{1}{n} \sum_{i=1}^{n} H(X_{i} \mid Y_{\sim i}(\cp))$
705: is the ({\em average}) $\exit$ function.\footnote{\label{foo:exit} More precisely,
706: $\exit$ functions are usually defined as
707: $I(X_i \mid Y_{\sim i}(\epsilon))=H(X_i)-H(X_i \mid Y_{\sim i}(\epsilon))$, which differs
708: from our definition only in a trivial way.}
709: We conclude that {\em for transmission over $\EC(\cp)$},
710: $h^{}(\cp) = \frac{1}{n} \frac{\text{d} \phantom{\cp}}{\text{d} \cp} H(X \mid Y(\cp))$.
711: If we integrate this relationship with respect to
712: $\cp$ from $0$ to $1$ and note that
713: $H(X \mid Y(0))=0$ and
714: $H(X \mid Y(1))=H(X)$, then we get the basic form of the area theorem
715: for the $\EC(\cp)$:
716: $\int_{0}^{1} h^{}(\cp) \text{d} \cp = H(X)/n$.
717: This statement was first proved, in the binary case, by Ashikhmin, Kramer, and
718: ten Brink in \cite{AKtB04} using a different framework.
719: \bex[Area Theorem for Repetition Code and $\BEC$]
720: Consider the {\em binary} repetition code with parameters $[\n, 1, \n]$,
721: where the first component describes the blocklength, the second
722: component denotes the dimension of the code, and the final component
723: gives the minimum (Hamming) distance.
724: By symmetry $h^{}_i(\ent)=h^{}(\ent)=
725: \ent^{n-1}$ for all $i \in [n]$.
726: We have $\int_{0}^{1} h^{}(\ent) \text{d} \ent=\frac{1}{n}=H(X)/n$,
727: as predicted.
728: \eex
729: The above scenario can easily be generalized by allowing the various components
730: of the code to be transmitted over different erasure channels. Consider, e.g.,
731: a binary repetition code of length
732: $n$ in which the first component is transmitted through $\BEC(\delta)$,
733: where $\delta$ is constant, but the remaining components are passed through
734: $\BEC(\ent)$. In this case we have
735: $\int_{0}^{1} h^{}(\ent) \text{d} \ent=
736: (H(X \mid Y(\delta, 1, \cdots, 1))-H(X \mid Y(\delta, 0, \cdots, 0)))/n=\delta/n$
737: (assuming that $X$ is chosen uniformly at random from the set of codewords).
738: We will get back to this point shortly when we introduce $\gexit$ functions
739: in Definition \ref{def:gexit}.
740:
741: The concept of $\exit$ functions
742: extends to general channels in the natural way.
743: To simplify notation somewhat let us focus on the binary case.
744: \bdefi[$\exitf {}$ for $\BMS$ Channels]
745: Let $X$ be a binary vector of length $n$ chosen with probability $p_X(x)$.
746: Assume that transmission takes place over the family $\{\BMS(\ent)\}_{\ent}$.
747: Then
748: \begin{align*}
749: \exitfi {}(\ent) & \defas H(X_i \mid Y_{\sim i}(\ent)), \\
750: \exitf {}(\ent) & \defas \frac{1}{n} \sum_{i=1}^{n} H(X_i \mid Y_{\sim i}(\ent))
751: = \frac{1}{n} \sum_{i=1}^{n} \exitfi {}(\ent).
752: \end{align*}
753: \edefi
754: This is the definition of the $\exit$ function introduced by ten Brink
755: \cite{teB99a,teB99b,tBr00,teB00,teB01} (see footnote \ref{foo:exit}).
756:
757: We get a more explicit representation if we consider transmission
758: using binary {\em linear} codes. In this context recall that a binary
759: linear code is {\em proper} if it possess a generator matrix with no zero columns.
760: As a consequence, in a proper binary linear code half the codewords
761: take on the value
762: $+1$ and half the value $-1$ in each given position.
763: \blemma[$\exitf {}$ for Linear Codes and $\BMS$ Channels]
764: \label{lem:exitlinear}
765: Let $X$ be chosen uniformly at random from a proper binary linear code
766: and assume that transmission takes place over the family
767: $\{\BMS(\ent)\}_{\ent}$.
768: Define
769: \begin{align}
770: \label{equ:phidef}
771: \phi_i(y_{\sim i}) & \defas
772: \lognat
773: \Bigl(
774: \frac{p_{X_i \mid Y_{\sim i}}(+1 \mid y_{\sim i})}{p_{X_i \mid Y_{\sim i}}(-1 \mid y_{\sim i})}
775: \Bigr),
776: \end{align}
777: and $\Phi_i \defas \phi_i(Y_{\sim i})$.
778: Let $\Ldens{a}_i$ denote the density of $\Phi_i$,
779: assuming that the all-one codeword was transmitted, and let
780: $\Ldens{a} \defas \frac{1}{n} \sum_{i=1}^{n} \Ldens{a}_i$.
781: Then
782: \begin{xalignat*}{3}
783: \exitfi {}(\ent) & =
784: \entropy(\Ldens{a}_i), &
785: \exitf {}(\ent) & =
786: \entropy(\Ldens{a}),
787: \end{xalignat*}
788: where $\entropy(\cdot)$ is the entropy operator
789: introduced in (\ref{equ:entropy}).
790: \elemma
791: \begin{proof}
792: Note that $X_i \rightarrow \Phi_i \rightarrow Y_{\sim i}$ forms
793: a Markov chain.\footnote{
794: For $z \in \reals$, let $y_{\sim i}$ be an element of
795: $\bigl(\phi_i\bigr)^{-1}(z)$ so that
796: $z = \phi_i(y_{\sim i})$. Then
797: $p_{X_i \mid Y_{\sim i}, \Phi_i}(x_i \mid y_{\sim i}, z)
798: = \frac{(1+x_i) +(1-x_i) e^{z}}{2(1+e^{z})}=
799: p_{X_i \mid \Phi_i}(x_i \mid z)$.
800: From this we conclude that
801: $p_{Y_{\sim i} \mid X_i, \Phi_i}(y_i \mid x_{\sim i}, z)=
802: p_{Y_{\sim i} \mid \Phi_i}(y_{\sim i} \mid z)$.}
803: Equivalently, we claim that $\Phi_i$ is a sufficient
804: statistic for $X_i$. From this we conclude that (see \cite[Section 2.8]{CoT91})
805: \begin{align*}
806: H(X_i \mid Y_{\sim i}) & = H(X_i \mid \Phi_i).
807: \end{align*}
808: Now note that since we assume that $X$ was chosen
809: uniformly at random from a proper binary linear codes, it follows
810: that the prior for each $X_i$ is the uniform one.
811: Therefore, $\Phi_i$ is in fact a log-likelihood ratio.
812: It is shown in \cite[Lemma 3.37]{RiU05} that, assuming that $X$ is chosen
813: uniformly at random from a proper binary linear code, the binary
814: ``channel'' $p(\phi_i \mid x_i)$ is symmetric.
815: %Further, note
816: %the $L$-density of $\Phi_i$ conditioned on $X_i=1$
817: %is equal to the density of $\Phi_i$ conditioned on $X_i=1$.
818: Further, note that the density of $\Phi_i$ conditioned on $X_i=1$
819: is equal to the density of $\Phi_i$
820: conditioned that the all-one codeword was transmitted.\footnote{To see this,
821: note that, using the symmetry of the channel and the equal prior on the codewords,
822: we can write
823: $p_{X_i\mid Y}(x_i \mid y) = c(y)
824: \sum_{\tilde{x} \in {\cal C}: \tilde{x}_i=x_i}
825: p_{Y \mid X}(y \tilde{x} \mid \underline{1})$,
826: where $c(y)$ is a constant independent of $x_i$, ${\cal C}$ denotes
827: the code, and $\underline{1}$ denotes
828: the all-one codeword.
829: In the same manner, if $x' \in {\cal C}$, then
830: $p_{X_i\mid Y}(x_i \mid y x') = c'(y)
831: \sum_{\tilde{x} \in {\cal C}: \tilde{x}_i=x_i x_i'}
832: p_{Y \mid X}(y x' \tilde{x} \mid x')$.
833: Compare the density of the log-likelihood ratio
834: assuming that the all-one codeword was transmitted to the one assuming that
835: the codeword $x'$ was transmitted. The claim follows by noting that
836: for any $y \in {\cal Y}$, $p_{Y \mid X}(y \mid \underline{1})=
837: p_{Y \mid X}(y x' \mid x')$, and that in this case also
838: $p_{Y \mid X}(y \tilde{x} \mid \underline{1})=
839: p_{Y \mid X}(y x' \tilde{x} \mid x')$.
840: }
841: By assumption this $L$-density is equal to $\Ldens{a}_i$.
842: We conclude that $H(X_i \mid \Phi_i)=\entropy(\Ldens{a}_i)$.
843: \end{proof}
844:
845: As the next example shows, the $\exit$ function does {\em not}
846: fulfill the area theorem in the general case.
847: \bex[$\exit$ Function for General $\BMS$ Channels]
848: Fig.~\ref{fig:exitchartbecexamples} shows the $\exit$ function for
849: the $[3, 1, 3]$ repetition code as well as for the
850: $[6, 5, 2]$ single parity-check code for $\BEC(\ent)$,
851: BSC$(\ent)$, and BAWGNC$(\ent)$. E.g., the $\exit$ function
852: for the $[\n,\n-1,2]$ single parity-check code over BSC$(\ent)$
853: is given by
854: \begin{align*}
855: \exitfi {}(\ent) = \exitf {}(\ent) = h_2\Bigl(\frac{1-(1-2\cp(\ent))^{\n-1}}{2} \Bigr),
856: \end{align*}
857: where
858: $\cp(\ent)=h_2^{-1}(\ent)$. Note that these $\exit$ functions
859: are ``ordered.'' More precisely, for a repetition code we get the highest
860: extrinsic entropy at the output for the channel
861: family $\{ \text{BSC}(\ent)\}_{\ent}$ and we get the lowest such entropy if
862: we use instead the family $\{ \text{BEC}(\ent)\}_{\ent}$. Indeed, one can show
863: that these two families are the {\em least} and {\em most} ``informative''
864: family
865: of channels over the whole class of $\BMS$ channels for a repetition code,
866: \cite{LHHH03,HuH03,SSZ03}.
867: The roles are exactly exchanged at a check node.
868: Since we know
869: that the $\exit$ function for the $\BEC$ fulfills the area theorem, it follows
870: from this extremality properties that the $\exit$ functions
871: for the $\BSC$ and
872: the $\BAWGNC$ do {\em not} fulfill the area theorem.
873: Indeed, for a single parity-check code with $n=3$ and the $\BSC(\ent)$ the area under
874: the $\exit$ function is given by
875: \begin{align*}
876: \int_{0}^{1} h_2\left(\frac{1-(1-2\cp(\ent))^{2}}{2} \right) \;\text{d}\ent \approx 0.643704 < 2/3.
877: \end{align*}
878: \begin{figure}[htp]
879: \centering
880: \setlength{\unitlength}{0.75bp}%
881: \begin{picture}(130,130)
882: \put(0,0){\includegraphics[scale=0.75]{exitchartbmsexamples}}
883: \put(-2, -2){\makebox(0,0)[rt]{\small $0$}}
884: \put(120,-2){\makebox(0,0)[t]{\small $1$}}
885: \put(-2,120){\makebox(0,0)[r]{\small $1$}}
886: \put(2,122){\makebox(0,0)[bl]{\small{$\exitf {}$}}}
887: \put(122, 2){\makebox(0,0)[bl]{\small{$\ent$}}}
888: \put(47,85){\makebox(0,0)[l]{\small $[6, 5, 2]$}}
889: \put(71,25){\makebox(0,0)[l]{\small $[3, 1, 3]$}}
890: \end{picture}
891: \caption{\label{fig:exitchartbecexamples} The $\exit$ function of
892: the $[3, 1, 3]$ repetition code and the $[6, 5, 2]$ parity-check code
893: for the $\BEC(\ent)$ (solid curve),
894: BSC$(\ent)$ (dashed curve) and BAWGNC$(\ent)$ (dotted
895: curve).}
896: \end{figure}
897: \eex
898: Although the above fact might be disappointing it is not surprising.
899: As it should be clear from the discussion at the beginning of
900: this section, the $\exit$ function is related to the GAT
901: only in the case of the erasure channel.
902: Let us therefore go back to the GAT and
903: {\em define} the function which fulfills the area theorem
904: in the general case.
905: \bdefi[$\gexit$ Function]
906: \label{def:gexit}
907: Let $X$ be a vector of length $n$ chosen with
908: probability $p_X(x)$ from ${\cal X}^n$.
909: Let the channel from $X$ to $Y$ be memoryless, where
910: $Y_i$ is the result of passing $X_i$
911: through the smooth
912: family $\{\text{M}(\cp_i)\}_{\cp_i}$, $\cp_i \in [0, 1]$.
913: Assume that all individual channels are parameterized
914: in a smooth (differentiable) way
915: by a common parameter $\cp$, i.e., $\cp_i=\cp_i(\cp)$, $i \in [n]$.
916: Let $\Omega$ be a further observation of $X$ so that
917: $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
918: Then the $i^{\text{th}}$ and the (average) {\em generalized} $\exit$ ($\gexit$) function
919: are defined by
920: %
921: \begin{align*}
922: %
923: \gexitfi {}(\cp) & \defas
924: \frac{\partial H(X_i | Y, \Omega)}{\partial \cp_i} \frac{\text{d} \cp_i}{\text{d} \cp} \Big|_{\cp}, \\
925: \gexitf {}(\cp) & \defas \frac{1}{\n}\sum_{i=1}^{\n} \gexitfi {}(\cp).
926: %
927: \end{align*}
928: %
929: %
930: \edefi
931: Discussion: The definition is stated in quite general terms.
932: First note that if we consider the
933: integral
934: $\int_{\underline{\cp}}^{\overline{\cp}} \gexitf {}(\cp) \text{d}\cp$,
935: then from Theorem \ref{theo:generalareatheorem} we conclude that
936: the result is
937: $\frac{1}{n}\bigl(H(X \mid Y(\overline{\cp}), \Omega)-H(X \mid Y(\underline{\cp}), \Omega)\bigr)$.
938: In words, if we smoothly change the individual channel parameters $\cp_i$
939: as a function of $\cp$,
940: then the integral of $\gexitfi {}(\cp)$
941: tells us how much the conditional entropy of the system changes
942: due to the total change of the parameters $\cp_i$. To be concrete,
943: assume, e.g., that all bits are sent through Gaussian channels.
944: We can imagine that we first only change the parameter of the
945: Gaussian channel through which bit $1$ is sent from its initial
946: to its final value, then the parameter of the second channel and so
947: on. Alternatively, we can imagine that all channel parameters are
948: changed simultaneously. In the two cases the integrals of the
949: individual $\gexit$ functions $\gexitfi {}$
950: differ but their sum is the same and it equals the
951: total change of the conditional entropy due to the change
952: of channel parameters. Therefore, $\gexit$ functions can
953: be considered to be a ``local'' way of measuring
954: the change of the conditional entropy of a system.
955: One should think of the common parameter $\cp$
956: as a convenient way of parameterizing the
957: path through ``channel space'' that we are taking.
958:
959: In many applications all channels are identical,
960: and formulas simplify significantly.
961: In Section \ref{sec:egexit} we will see a case in which the extra degree
962: of freedom afforded by allowing different channels is important.
963: The
964: additional observation $\Omega$ is useful
965: if we consider the design or iterative systems and component-wise
966: $\gexit$ functions. For what follows though we will not need it. Hence,
967: we will drop $\Omega$ in the sequel.
968:
969: If we assume that the input is {\em binary}
970: we obtain a more explicit expression for the $\gexit$ functions.
971: \blemma[$\gexitf {}$ for $\BM$ Channels]\label{lemma:gexitBM}
972: Let $X$ be a {\em binary} vector of length $n$ chosen with
973: probability $p_X(x)$.
974: Let the channel from $X$ to $Y$ be memoryless, where
975: $Y_i$ is the result of passing $X_i$ over
976: the smooth family $\{\text{BM}(\ent_i)\}_{\ent_i}$, $\ent_i \in [0, 1]$.
977: Assume that all individual channel families are parameterized
978: in a smooth (differentiable) way
979: by a common parameter $\cp$, i.e., $\ent_i=\ent_i(\cp)$, $i \in [n]$.
980: Then the $i^{\text{th}}$ and the (average) {\em generalized} $\exit$ ($\gexit$) function
981: are given by
982: \begin{align}
983: \gexitfi {}(\cp) & =
984: \int_{\phi_i,y_i}
985: \sum_{x_i}
986: p(x_i)
987: p(\phi_i \mid x_i)
988: \frac{\text{d}\phantom{\ent}}{\text{d} \ent_i} p(y_i|x_i)\cdot
989: \label{equ:gexitcompact0} \\
990: & \phantom{xxxx} \cdot
991: \log\left\{\sum_{x'_i}\frac{p(x'_i|\phi_i)
992: p(y_i|x'_i)}{p(x_i|\phi_i)p(y_i|x_i)}
993: \right\} \frac{\text{d} \ent_i}{\text{d} \cp} \text{d} y_i \text{d} \phi_i, \nonumber \\
994: \gexitf {}(\cp) & = \frac{1}{n} \sum_{i=1}^{n} \gexitfi {}(\cp) ,
995: \label{equ:gexitcompact}
996: \end{align}
997: where $\phi_i(y_{\sim i})$ and $\Phi_i$ are
998: defined as in (\ref{equ:phidef}).
999: \elemma
1000: Discussion: As mentioned above, the derivative
1001: of $p(y_i|x_i)$ in Eq.~(\ref{equ:gexitcompact0}) has to be interpreted in
1002: general as in Eq.~(\ref{eq:FormalDerivative}). Moreover, writing the
1003: same expression as $\gexitfi {}(\cp) =\int f(y)
1004: \frac{\text{d}\phantom{\ent}}{\text{d} \ent_i} p(y_i|x_i)\de y$,
1005: the existence of such derivative follows from the channel
1006: smootheness and the differentiability of $f(y)$ (if written as a
1007: function of the log-likelihood $\log\frac{p(y|+1)}{p(y|-1)}$.
1008:
1009: \begin{proof}
1010: We proceed as in the proof of Lemma \ref{lem:exitlinear}.
1011: We claim that $X_i \rightarrow (\Phi_i, Y_i) \rightarrow Y$ forms
1012: a Markov chain (equivalently, $(\Phi_i, Y_i)$ constitutes
1013: a sufficient statistic).
1014: To see this, fix $z \in \reals$ and let $y_{\sim i}$ be an element of
1015: $\bigl(\phi_i\bigr)^{-1}(z)$, so that
1016: $z = \phi_i(y_{\sim i})$. Then,
1017: using the fact that $Y_i$ is conditionally independent of $Y_{\sim i}$,
1018: given $X_i=x_i$, we may write
1019: %
1020: \begin{align*}
1021: %
1022: p_{X_i \mid Y_i, Y_{\sim i}, \Phi_i}(x_i \mid y_i, y_{\sim i}, z)
1023: =\\ \frac{p_{Y_i\mid X_i}(y_i\mid x_i)
1024: p_{X_i\mid Y_{\sim i}, \Phi_i}(x_i\mid y_{\sim i},z)}
1025: {\sum_{x_i'\in{\cal X}}
1026: p_{Y_i\mid X_i}(y_i|x_i')p_{X_i\mid Y_{\sim i}, \Phi_i}
1027: (x_i'\mid y_{\sim i},z)}
1028: %
1029: \end{align*}
1030: %
1031: Since $X_i\to \Phi_i\to Y_{\sim i}$ forms a Markov chain
1032: (as already shown in the proof of Lemma \ref{lem:exitlinear}),
1033: we have $p_{X_i\mid Y_{\sim i}, \Phi_i}(x_i\mid y_{\sim i},z)=
1034: p_{X_i\mid \Phi_i}(x_i\mid z)$.
1035: Substituting in the above equation, we get
1036: $p_{X_i \mid Y_i, Y_{\sim i}, \Phi_i}(x_i \mid y_i, y_{\sim i}, z)
1037: = p_{X_i \mid Y_i, \Phi_i}(x_i \mid y_i, z)$, as claimed.
1038:
1039: Therefore, we can rewrite $\gexitfi {}(\cp)$ as
1040: \begin{align*}
1041: \gexitfi {}(\cp)
1042: & =
1043: \frac{\partial H(X_i | Y)}{\partial \ent_i} \frac{\text{d} \ent_i}{\text{d} \cp} \Big|_{\cp}
1044: = \frac{\partial H(X_i | \Phi^{}_i, Y_i)}{\partial \ent_i}
1045: \frac{\text{d} \ent_i}{\text{d} \cp} \Big|_{\cp}.
1046: \end{align*}
1047: Expand $H(X_i | \Phi^{}_i, Y_i)$ as
1048: %
1049: \begin{align*}
1050: %
1051: & -\int_{\phi_i, y_i} \sum_{x_i} p(x_i, \phi_i, y_i)
1052: \logtwo(p(x_i \mid \phi_i, y_i)) \text{d} y_i \text{d} \phi_i \\
1053: &= -\int_{\phi_i, y_i}\sum_{x_i}
1054: p(x_i) p(\phi_i \mid x_i) p(y_i|x_i)\cdot \\
1055: & \phantom{xxxx}
1056: \cdot\logtwo\left\{\frac{p(x_i|\phi_i)p(y_i|x_i)}{\sum_{x_i'\in\cX}
1057: p(x'_i|\phi_i)p(y_i|x'_i)}\right\} \text{d} y_i \text{d} \phi_i .\nonumber
1058: %
1059: \end{align*}
1060: %
1061: This form has the advantage that the dependence of
1062: $H(X_i | \Phi_i, Y_i)$ upon the channel at position $i$
1063: is completely explicit. Let us therefore differentiate the
1064: above expression with respect to $\ent_i$, the parameter
1065: which governs the transition probability $p(y_i \mid x_i)$. The terms obtained
1066: by differentiating with respect to the channel {\em inside} the $\logtwo$
1067: vanish. For instance, when differentiating with respect to the $p(y_i|x_i)$
1068: at the numerator, we get
1069: %
1070: \begin{align*}
1071: & -\int_{\phi_i, y_i}
1072: \sum_{x_i}
1073: p(x_i) p(\phi_i \mid x_i)
1074: \frac{\text{d}\phantom{\ent}}{\text{d} \ent_i}
1075: p(y_i \mid x_i) \text{d} y_i \text{d} \phi_i \\
1076: = & -\int_{\phi_i}
1077: \sum_{x_i} p(x_i) p(\phi_i \mid x_i)
1078: \frac{\text{d}\phantom{\ent}}{\text{d} \ent_i}\int_{y_i} p(y_i \mid x_i)
1079: \text{d} y_i \text{d} \phi_i
1080: =0\, .
1081: \end{align*}
1082: When differentiating with respect to the {\em outer} $p(y_i|x_i)$
1083: we get the stated result.
1084: \end{proof}
1085: Although the last lemma was stated for the case of binary channels, it poses
1086: no difficulty to generalize it. It is in fact sufficient to replace
1087: $\phi_i(y_{\sim i})$ with any sufficient statistic
1088: of $X_i$, given $Y_{\sim i}=y_{\sim i}$. For instance, one may take
1089: $\phi_i(y_{\sim i}) = \{p_{X_i|Y_{\sim i}}(x_i\mid y_{\sim i}) ;\;
1090: x_{i}\in{\cal X}\}$, which takes value on the
1091: $(|{\cal X}|-1)$-dimensional simplex, or any parameterization of it.
1092: The log-likelihood can be regarded as a particular parameterization
1093: of the $1$-dimensional simplex. More generally,
1094: $p_{X_i|Y_{\sim i}}(x_i\mid y_{\sim i})$ is a natural quantity appearing
1095: in iterative decoding. The proof (as well as the statement) applies verbatimly
1096: to this case.
1097:
1098: We get an even more compact description if we assume that
1099: transmission takes place using a binary {\em proper linear} code
1100: and that the channel is {\em symmetric}.
1101: %
1102: \blemma[$\gexitf {}$ for Linear Codes and $\BMS$ Channels]
1103: \label{lem:gexitlinear}
1104: Let $X$ be chosen uniformly at random from a proper binary linear code of
1105: length $n$. Let the channel from $X$ to $Y$ be memoryless, where
1106: $Y_i$ is the result of passing $X_i$ over the smooth family
1107: $\{\BMS(\ent_i)\}_{\ent_i}$.
1108: Assume that all individual channels are parameterized
1109: in a smooth (differentiable) way by a common parameter $\cp$, i.e., $\ent_i=\ent_i(\cp)$, $i \in [n]$.
1110: Let the $i^{\text{th}}$ channel be characterized by its $L$-density, which by
1111: some abuse of notation we denote by
1112: $\Ldens{c}_{\BMSsmall(\ent_i)}$.
1113: Let $\phi_i$ and $\Phi_i$ be as defined
1114: in (\ref{equ:phidef}) and let $\Ldens{a}_i$ denote the density of
1115: $\Phi_i$, assuming that the all-one codeword was transmitted.
1116: Then
1117: \begin{align*}
1118: \gexitfi {}(\cp) & = \int_{-\infty}^{\infty}
1119: \Ldens{a}_i(z) \gexitkl {\Ldens{c}_{\BMSsmall(\ent_i)}} z \;\text{d}z,
1120: \end{align*}
1121: where
1122: \begin{align*}
1123: \gexitkl {\Ldens{c}_{\BMSsmall(\ent_i)}} z & \defas
1124: \int_{-\infty}^{\infty}
1125: \frac{\partial \Ldens{c}_{\BMSsmall(\ent_i)}(w)}{\partial \cp }
1126: \logtwo(1+ \mbox{e}^{-z-w} ) \;\text{d}w.
1127: \end{align*}
1128: \elemma
1129: Discussion: The remarks made after Lemma \ref{lemma:gexitBM}
1130: apply in particular to the present case:
1131: We write
1132: $\int_{-\infty}^{\infty}
1133: \frac{\partial \Ldens{c}_{\BMSsmall(\ent_i)}(w)}{\partial \cp }
1134: \logtwo(1+ \mbox{e}^{-z-w} ) \;\text{d}w$ as a proxy
1135: for $\frac{\partial}{\partial \cp}
1136: \left\{\int_{-\infty}^{\infty}
1137: \Ldens{c}_{\BMSsmall(\ent_i)}(w)
1138: \logtwo(1+ \mbox{e}^{-z-w} ) \;\text{d}w \right\}$. The latter
1139: expression exists, since
1140: $\logtwo(1+ \mbox{e}^{-z-w} )$ is continuously differentiable
1141: as a function of $w$ and by assumption the channel
1142: family is smooth.
1143: Note further that $\gexitkl {\Ldens{c}_{\BMSsmall(\ent_i)}} z$
1144: is continuous and non-negative so that $\gexitfi {}(\cp)$ exists as well.
1145:
1146: \begin{proof}
1147: Consider the expression for $\gexitfi {}(\cp)$
1148: as given in (\ref{equ:gexitcompact}). By assumption,
1149: $p(y_i \mid x_i)$ is symmetric for all $i \in [n]$. Further, as
1150: already remarked in the proof of Lemma \ref{lem:exitlinear}, the ``channel''
1151: $p(\phi_i \mid x_i)$ is symmetric as well.
1152: It follows from this and the fact that $p_{X_i}(+1)=p_{X_i}(-1)$ (due to
1153: the assumption that the code is proper and that codewords are chosen
1154: with uniform probability) that the contributions to
1155: $\gexitfi {}(\cp)$ for $x_i=+1$ and $x_i=-1$ are identical.
1156: We can therefore assume without loss of generality that $x_i=+1$.
1157: Recall that the density of $\Phi_i$ assuming that $X_i=1$
1158: is equal to the density of $\Phi_i$ assuming that the all-one
1159: codeword was transmitted. The latter is by definition equal to
1160: $\Ldens{a}_i$. As remarked earlier, $\Ldens{a}_i$ is
1161: symmetric. Further, as discussed in the introduction, we can
1162: assume that the $i^{\text{th}}$ $\BMS$ channel outputs already log-likelihood
1163: ratios. Therefore, $p_{Y_i | X_i}(y_i |+1)=\Ldens{c}_{\BMSsmall(\ent_i)}(y_i)$.
1164: Finally, consider the expression within the $\logtwo$. If $x_i'=+1$
1165: then the numerator and denominator are equal and we get one.
1166: If on the other hand $x_i'=-1$ then we get by the previous remarks
1167: the product of the likelihoods. Putting this all together we get
1168: \begin{align*}
1169: \gexitfi {}(\cp) & =
1170: \int
1171: \Ldens{a}_i(z)
1172: \frac{\text{d} \Ldens{c}_{\BMSsmall(\ent_i)}(w)}{\text{d} \ent_i}
1173: \logtwo\left(1+\text{e}^{-z-w} \right) \text{d}z \text{d} w.
1174: \end{align*}
1175: The thesis follows by rearranging terms.
1176: \end{proof}
1177:
1178: \bex[Alternative Kernel Representations] \label{ex:nonuniquekernel}
1179: Note that because of the symmetry property of $L$-densities we can
1180: write
1181: \begin{align*}
1182: \gexitf {}(\cp) & = \int_{-\infty}^{\infty}
1183: \Ldens{a}(z) \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z \;\text{d}z \\
1184: & = \int_{0}^{\infty}
1185: \absLdens{a}(z)
1186: \frac{\gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z +
1187: e^{-z} \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} {-z} }{1+e^{-x}}\text{d}z.
1188: \end{align*}
1189: This means that the kernel is uniquely specified on the absolute value domain
1190: $[0, \infty]$, but that for each $z \in [0, \infty]$
1191: we can split the weight of the kernel in any desired way between $+z$ and $-z$
1192: so that $\gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z +
1193: e^{-z} \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} {-z}$ equals the desired value.
1194: In the sequel we will use this degree of freedom to bring some
1195: kernels into a more convenient form. Although it constitutes some
1196: abuse of notation we will in the sequel make no notational distinction
1197: between equivalent such kernels even though pointwise they might not
1198: represent the same function.
1199: \eex
1200:
1201: As we have already remarked in the discussion
1202: right after Definition \ref{def:gexit}, the $\gexit$ functions
1203: $\gexitfi {}(\cp)$ allow us to ``locally'' measure
1204: the change of the conditional entropy of a system.
1205: This property is particularly apparent in the representation of
1206: Lemma \ref{lem:gexitlinear} where we see that the local measurement has two components:
1207: (i) the kernel which depends on the derivative of the channel
1208: seen at the given position and (ii) the distribution $\Ldens{a}_i$,
1209: which encapsulates all our ignorance
1210: about the code behavior with respect to the $i^{\text{th}}$ position.
1211: This representation is very intuitive. If we improve the observation
1212: of a particular bit (derivative of the channel with respect to the parameter)
1213: then the amount by which the conditional entropy
1214: of the overall system changes clearly depends on how well this particular
1215: bit was already known via the code constraints and
1216: the observations of the other bits (extrinsic posterior density): if
1217: the bit was already perfectly known then the additional observation
1218: afforded will be useless, whereas if nothing was known about the bit
1219: one would expect that the additional reduction in entropy of this bit
1220: fully translates to a reduction of the entropy of the overall system.
1221: We will see some quantitative statements of this nature in Section
1222: \ref{sec:basicproperties}.
1223: %In particular the density $\Ldens{a}_i$
1224: %is in turn the natural object appearing in message passing
1225: %algorithms and in the density evolution analysis.
1226:
1227: In the next three examples we compute the kernels
1228: $\gexitkl {\Ldens{c}_{\BMSsmall(\ent_i)}} z$ for the standard
1229: families $\{\BEC(\ent)\}_{\ent}$, $\{\text{BSC}(\ent)\}_{\ent}$, and
1230: $\{\text{BAWGNC}(\ent)\}_{\ent}$. If we consider a single family of
1231: $\BMS$ channels parameterized by the entropy $\ent$ it is
1232: convenient to ``normalize'' the $\gexit$ kernel so that it
1233: measure the ``progress per $\text{d} \ent$''. This means, in
1234: the following examples we compute
1235: \begin{align}
1236: \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z & \defas
1237: \frac{
1238: \int_{-\infty}^{\infty}
1239: \frac{\partial \Ldens{c}_{\BMSsmall(\ent_i)}(w)}{\partial \cp }
1240: \logtwo(1+ \mbox{e}^{-z-w} ) \;\text{d}w}{
1241: \int_{-\infty}^{\infty}
1242: \frac{\partial \Ldens{c}_{\BMSsmall(\ent_i)}(w)}{\partial \cp }
1243: \logtwo(1+ \mbox{e}^{-w} ) \;\text{d}w}. \label{kernelcomputation}
1244: \end{align}
1245:
1246: \bex[$\gexit$ Kernel, $L$-Domain -- $\{\BEC(\ent)\}_{\ent}$] \label{ex:lbeckernel}
1247: If we take the family $\{\Ldens{c}_{\BECsmall(\ent)}\}_{\ent}$,
1248: where $\ent=\epsilon$ denotes both, the channel (intrinsic) entropy
1249: and the cross-over erasure probability,
1250: then a quick calculation
1251: shows that $\gexitkl {\Ldens{c}_{\BECsmall(\ent)}} z =
1252: \log_2(1+e^{-z})=\exitkl z$. In words, the $\gexit$ kernel
1253: with respect to the family
1254: $\{\BEC(\ent)\}_{\ent}$ is the regular $\exit$ kernel.
1255: \eex
1256:
1257: \bex[$\gexit$ Kernel, $L$-Domain -- $\{\text{BSC}(\ent)\}_{\ent}$]
1258: \label{ex:lbsckernel}
1259: Let us now look at the family
1260: $\{\Ldens{c}_{\BSCsmall(\ent)}\}_{\ent}$. Some calculus shows that
1261: \begin{align*}
1262: \gexitkl {\Ldens{c}_{\BSCsmall(\ent)}} z & =
1263: \log\left( \frac{1+\frac{1-\cp}{\cp}\mbox{e}^{-z}}{1+\frac{\cp}{1-\cp}\mbox{e}^{-z} }\right)/\log\left(\frac{1-\cp}{\cp} \right),
1264: \end{align*}
1265: where $\cp = h_2^{-1}(\ent)$. For a fixed $z \in \reals$ and
1266: $\ent\to 0$, the kernel converges to $1$ as $1+z/\lognat(\cp)$, whereas the limit when $\ent\to 1$
1267: is equal to $\frac{2}{1+e^{z}}$.
1268: \eex
1269:
1270: \bex[$\gexit$ Kernel, $L$-Domain --
1271: $\{\text{BAWGNC}(\ent)\}_{|ent}$]
1272: \label{ex:lbawgnkernel}
1273: Consider now the family
1274: $\{\Ldens{c}_{\BAWGNCsmall(\ent)}\}_{\ent}$, where $\ent$ denotes
1275: the channel entropy. This family is defined in Example
1276: \ref{ex:channeldefinition}.
1277: Recall that the noise is assumed to be Gaussian with
1278: zero-mean and variance $\sigma^2$.
1279: A convenient parameterization for this case is
1280: $\cp\defas2/\sigma^2$. This means that in the following
1281: $\ent = \entropy(\Ldens{c}_{\text{\tiny BAWGNC}(\sigma^2=2/\cp)})$.
1282: After some steps of calculus shown
1283: in Appendix \ref{app:gaussiankernel} and Lemma \ref{lemma:equivkernel}, we get
1284: \begin{align*}
1285: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z
1286: & =
1287: \left({ \int_{-\infty}^{+\infty}\frac{\scriptstyle \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{ \scriptstyle 1 + \text{e}^{w+z} }\text{d}w }\right)/\left( {\int_{-\infty}^{+\infty}\frac{ \scriptstyle \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{ \scriptstyle 1 + \text{e}^{w} } \text{d}w}\right).
1288: \end{align*}
1289: In Appendix \ref{app:gaussiankernel} we give alternative
1290: representations and/or interpretations of
1291: this kernel. In particular we discuss the relationship
1292: to the formulation presented by Guo, Shamai and Verd{\'u} in \cite{GSV04,GSV05}
1293: using a connection to the MSE detector
1294: as well as the formulation by Macris in \cite{Mac05}
1295: based on the Nishimori identity.
1296: \eex
1297: One convenient feature of standard $\exit$ functions is that
1298: they are fairly similar for a given code across the whole
1299: range of $\BMS$ channels. Is this still true for $\gexit$ functions?
1300: $\gexit$ functions depend on the channel {\em both} through the kernel as well
1301: as through the extrinsic densities.
1302: %The densities are the same as
1303: %for the computation of $\exit$ functions. But the kernels
1304: %are now also functions of the channel.
1305: Let us therefore compare the shape of the various kernels. It is most
1306: convenient to compare the kernels not in the $L$-domain but in
1307: the $|D|$-domain.
1308: A change of variables shows that in general the $L$-domain kernel,
1309: call it $\gexitkl {\Ldens{c}} \cdot$,
1310: and the associated $|D|$-domain
1311: kernel, denote it by $\gexitkabsd {\Ldens{c}} \cdot$, are linked by
1312: \begin{align}
1313: \label{equ:gexitkernelconversion}
1314: \gexitkabsd {\Ldens{c}} s & =
1315: \frac{1-s}{2} \gexitkl {\Ldens{c}} {\log\frac{1-s}{1+s}} +
1316: \frac{1+s}{2} \gexitkl {\Ldens{c}} {\log \frac{1+s}{1-s}}.
1317: \end{align}
1318: E.g.,
1319: if we apply the above transformation to the previous examples we
1320: get the following results.
1321:
1322: \bex[$\gexit$ Kernel, $|D|$-Domain -- $\{\text{BEC}(\ent)\}_{\ent}$]
1323: \label{ex:absdbeckernel}
1324: We get $\gexitkabsd {\Ldens{c}_{\BECsmall(\ent)}} s=h_2((1+s)/2)$.
1325: \eex
1326:
1327: \bex[$\gexit$ Kernel, $|D|$-Domain -- $\{\text{BSC}(\ent)\}_{\ent}$]
1328: \label{ex:absdbsckernel}
1329: Some calculus shows that
1330: $
1331: \gexitkabsd {\Ldens{c}_{\BSCsmall(\ent(\cp))}} s =1+ \frac{s}{\log((1-\cp)/\cp)}\log\left(\frac{1+2\cp s -s}{1-2\cp s+s}\right).
1332: $
1333: The limiting values are seen to be
1334: %\begin{align*}
1335: $\lim_{\ent\to1}\gexitkabsd {\Ldens{c}_{\BSCsmall(\ent)}} s = 1-s^2,$ and
1336: $\lim_{\ent\to0}\gexitkabsd {\Ldens{c}_{\BSCsmall(\ent)}} s = 1.$
1337: %\end{align*}
1338: \eex
1339:
1340: \bex[$\gexit$ Kernel, $|D|$-Domain -- $\{\text{BAWGN}(\ent)\}_{\ent}$]
1341: \label{ex:absdbawgnkernel}
1342: Using Example \ref{ex:lbawgnkernel} and
1343: (\ref{equ:gexitkernelconversion}),
1344: it is straightforward to write the kernel in the $|D|$-domain as
1345: \begin{align*}
1346: \gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent(\cp))}} s
1347: & = \sum_{i\in\{-1,+1\} }\frac
1348: { \int_{-\infty}^{+\infty}\frac{(1-s^2) \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{(1+is)+(1-is)\text{e}^w} \text{d}w}
1349: {\int_{-\infty}^{+\infty} \frac{2 \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{1+\text{e}^w }\text{d}w }.
1350: \end{align*}
1351: %
1352: As shown in Appendix \ref{app:kernellimits}, the limiting values are the
1353: same as for the BSC, i.e.,
1354: $\lim_{\ent\to1}\gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} s = 1-s^2$,
1355: and
1356: $\lim_{\ent\to0}\gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} s = 1$.
1357: \eex
1358:
1359: In Fig.~\ref{fig:gexitkernels} we compare
1360: the $\exit$ kernel (which is also the $\gexit$ kernel for the BEC)
1361: with the $\gexit$ kernels
1362: for BSC$(\ent)$ and BAWGNC$(\ent)$ in the
1363: $|D|$-domain for several channel parameters.
1364: Note that these kernels are distinct but quite similar.
1365: In particular, for $\ent=0.5$ the $\gexit$ kernel with respect to BAWGNC$(\ent)$
1366: is hardly distinguishable from the regular $\exit$ kernel.
1367: The $\gexit$ kernel for the $\BSC$ shows more variation.
1368: %For $\ent$ converging to zero the kernel in the $|D|$-domain converges to the
1369: % constant one for the BAWGN and the $\BSC$, whereas the limit when $\ent$
1370: %approaches one is equal to $1-s^2$ as explained in Examples \ref{ex:absdbsckernel}
1371: %and \ref{ex:absdbawgnkernel}.
1372: \begin{figure}[hbt]
1373: \centering
1374: \setlength{\unitlength}{0.4bp}
1375: \begin{picture}(600,200)
1376: \put(0,0)
1377: {
1378: \put(10,0){\includegraphics[scale=0.4]{gkld_0o1}} %length=180bp
1379: {\tiny
1380: %\footnotesize
1381: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1382: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1383: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1384: }
1385: }
1386: \put(200,0)
1387: {
1388: \put(10,0){\includegraphics[scale=0.4]{gkld_0o5}} %length=180bp
1389: {\tiny
1390: %\footnotesize
1391: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1392: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1393: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1394: }
1395: }
1396: \put(400,0)
1397: {
1398: \put(10,0){\includegraphics[scale=0.4]{gkld_0o9}} %length=180bp
1399: {\tiny
1400: %\footnotesize
1401: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1402: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1403: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1404: }
1405: }
1406:
1407: \end{picture}
1408: \caption{Comparison of the kernels
1409: $\gexitkabsd {\Ldens{c}_{\BECsmall(\ent)}} s$
1410: (dashed line) with
1411: $\gexitkabsd {\Ldens{c}_{\text{\tiny BSC}(\ent)}} s$
1412: (dotted line) and
1413: $\gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} s$
1414: (solid line) at channel entropy rate $\ent=0.1$ (left), $\ent=0.5$ (middle) and
1415: $\ent=0.9$ (right).}
1416: \label{fig:gexitkernels}
1417: \end{figure}
1418:
1419: \bex[Repetition Code]
1420: Consider the $[\n, 1, \n]$ repetition code.
1421: Let $\{\Ldens{c}_{\ent}\}_{\ent}$ characterize a smooth family of
1422: $\BMS$ channels. For $\n \in \naturals$, let $\Ldens{c}_{\ent}^{\conv \n}$ denote
1423: the $\n$-fold convolution of $\Ldens{c}_{\ent}$.
1424: The $\gexit$ function for the $[\n, 1, \n]$ repetition code is then given by
1425: $\gexitf {}(\ent) =
1426: \frac{1}{n} \frac{\text{d}\phantom{\ent}}{\text{d} \ent} \entropy(\Ldens{c}_{\ent}^{\conv n})$.
1427: Explicitly, we get $\gexitfc {} {\BECsmall}(\ent) = \ent^n =
1428: \exitfc {} {\BECsmall}(\ent)$.
1429: As a further example, $\gexitfc {} {\BSCsmall}$ is given in parametric form
1430: by
1431: \begin{align*}
1432: \Bigl(h_2(\cp), \frac{\sum_{j=\pm1} j\sum_{i=1}^{n} \binom{n}{i}
1433: \cp^i \overline{\cp}^{n-i} \log\bigl(1+(\cp/\overline{\cp})^{n-2i-j} \bigr)}{
1434: n \log\left({\overline{\cp}}/{\cp}\right)} \Bigr),
1435: \end{align*}
1436: with $\overline{\cp}=1-\cp.$
1437: \eex
1438: \bex[Single Parity-Check Code]
1439: Consider the dual code, i.e.,
1440: the $[\n, \n-1, 2]$ parity-check code. Some calculations show that
1441: $\gexitfc {} {\BSCsmall}$ is given in parametric form
1442: by
1443: \begin{align*}
1444: \Bigl(h_2(\cp), 1-(1-2 \cp)^{n-1}
1445: \frac{\log\bigl( \frac{1+(1-2 \cp)^{n}}{1-(1-2 \cp)^{n}} \bigr)}{\log \bigl(\frac{1-\cp}{\cp} \bigr)} \Bigr).
1446: \end{align*}
1447: No simple analytic expressions are known for the case of transmission
1448: over the \BAWGNC.
1449: \begin{figure}[hbt]
1450: \centering
1451: \setlength{\unitlength}{0.6bp}
1452: \begin{picture}(400,180)
1453: \put(0,0)
1454: {
1455: \put(10,0){\includegraphics[scale=0.6]{curveCollecBSC}} %length=180bp
1456: {\tiny
1457: %\footnotesize
1458: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1459: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1460: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1461: }
1462: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
1463: \put(22,172){\makebox(0,0)[lb]{$\exitf {}$, $\gexitf {}$}}
1464: }
1465: \put(200,0)
1466: {
1467: \put(10,0){\includegraphics[scale=0.6]{curveCollecBAWGN}} %length=180bp
1468: {\tiny
1469: %\footnotesize
1470: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1471: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1472: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1473: }
1474: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
1475: \put(22,172){\makebox(0,0)[lb]{$\exitf {}$, $\gexitf {}$}}
1476: }
1477: \end{picture}
1478: \caption{\label{fig:exitgexitchartbecexamples}
1479: The $\exit$ (dashed) and $\gexit$ (dotted) function of
1480: the $[\n, 1, \n]$ repetition code
1481: and the $[\n, \n-1, 2]$ parity-check code
1482: assuming that transmission takes place over BSC$(\ent)$ (left picture)
1483: or the BAWGNC$(\ent)$ (right picture), $\n\in\{2,3,4,5,6\}$.
1484: }
1485: \end{figure}
1486: \eex
1487: Fig.~\ref{fig:exitgexitchartbecexamples} compares EXIT to GEXIT curves
1488: for some repetition and some single parity-check codes.
1489: \bex[Hamming Code]
1490: Consider the $[7,4,3]$ Hamming code.
1491: When transmission takes place over BEC$(\ih)$,
1492: it is a tedious but conceptually simple exercise to
1493: show that the $\exit$ function is
1494: $\xh(\ih)=3 \ih^2+4\ih^3-15\ih^4+12\ih^5-3\ih^6$, see, e.g.,
1495: \cite{AKtB04,MeU03}.
1496: %The lowest degree of this polynomial in $\ih$
1497: %is equal to the minimum distance minus 1 of the linear code. Its degree
1498: %is $\n-1$.
1499: In a similar way, using the derivative of the
1500: conditional entropy, one can give an analytic expression for
1501: the $\gexit$ function assuming transmission takes place over the \BSC.
1502: Both expressions are evaluated in
1503: Fig.~\ref{fig:HamBesCgexit} (left).
1504: A comparison between $\gexit$ and $\exit$ functions
1505: for the Hamming code and the BSC is shown in
1506: Fig.~\ref{fig:HamBesCgexit} (right).
1507: \eex
1508: \bex[Simplex Code]
1509: Consider now the dual of the Hamming code, i.e., the $[7,3,4]$ Simplex code.
1510: %The same discussion as for its dual code holds.
1511: %The analytic expression
1512: For transmission over the \BEC\ we have
1513: $\xh(\ih)=4\ih^3-6\ih^5+3\ih^6$.
1514: %which has minimum degree $3=4-1$.
1515: Fig.~\ref{fig:HamBesCgexit}
1516: compares $\gexit$ and $\exit$ functions for this code when transmission
1517: takes place over the BEC and over the BSC.
1518: \eex
1519: \begin{figure}[hbt]
1520: \centering
1521: \setlength{\unitlength}{0.6bp}
1522: \begin{picture}(400,200)
1523: \put(0,0)
1524: {
1525: \put(10,0){\includegraphics[scale=0.6]{HamBesCgexit}} %length=180bp
1526: {\tiny
1527: %\footnotesize
1528: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1529: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1530: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1531: }
1532: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
1533: \put(22,172){\makebox(0,0)[lb]{$\exitf {}$, $\gexitf {}$}}
1534: \put(27,146){\makebox(0,0)[l]{\small $[7,4,3]$ Hamming}}
1535: \put(72,25){\makebox(0,0)[l]{\small $[7,3,4]$ Simplex}}
1536: }
1537: \put(200,0)
1538: {
1539: \put(10,0){\includegraphics[scale=0.6]{HamBsCgexit_exit}} %length=180bp
1540: {\tiny
1541: %\footnotesize
1542: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1543: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
1544: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
1545: }
1546: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
1547: \put(22,172){\makebox(0,0)[lb]{$\exitf {}$, $\gexitf {}$}}
1548: \put(27,146){\makebox(0,0)[l]{\small $[7,4,3]$ Hamming}}
1549: \put(72,25){\makebox(0,0)[l]{\small $[7,3,4]$ Simplex}}
1550: }
1551: \end{picture}
1552: \caption{Comparison of the $\gexit$ functions for the $[7,4,3]$ Hamming
1553: code and its dual. Left picture: Comparison between $\gexit$ functions
1554: when transmitting over the BEC (dashed line)
1555: and over the BSC (solid line). Right picture: Comparison between $\gexit$ (solid line) and $\exit$ (dashed line) functions
1556: when transmission takes place over the BSC.}
1557: \label{fig:HamBesCgexit}
1558: \end{figure}
1559:
1560: %
1561: %***********************************************************
1562: %
1563: \section{Basic Properties of $\gexit$ Functions}
1564: \label{sec:basicproperties}
1565: $\gexit$ functions fulfill the GAT by definition. Let us state a few more
1566: of their properties.
1567:
1568: We first show that the $\gexit$ function
1569: preserves the partial order implied by physical degradation.
1570: %
1571: \begin{lemma}
1572: \label{prop:physicaldegradationgeneral}
1573: %
1574: Let $X$ be chosen with probability $p_X(x)$ from ${\cal X}^n$.
1575: Let the channel from $X$ to $Y$ be memoryless, where
1576: $Y_i$ is the result of passing $X_i$ through the smooth and degraded family
1577: $\{\text{M}(\cp_i)\}_{\cp_i}$, $\cp_i \in I_i$.
1578: If $X \rightarrow Y_{\sim i} \rightarrow \Phi_i$ forms
1579: a Markov chain then
1580: \begin{align}
1581: \label{eq:generalphysicaldegradation}
1582: \frac{\partial H(X_i \mid Y) }{\partial \cp_i} \le
1583: \frac{\partial H(X_i \mid Y_i, \Phi_i) }{\partial \cp_i}.
1584: \end{align}
1585: %
1586: \end{lemma}
1587: %
1588: \begin{proof}
1589: %
1590: Since the derivatives in Eq.~ (\ref{eq:generalphysicaldegradation})
1591: are known to exist a.e., the above statement is in fact equivalent to saying that,
1592: for any $\cp'_i\ge\cp_i$,
1593: %
1594: \begin{align*}
1595: %
1596: H(X_i\mid Y_i(\cp_i'),Y_{\sim i})-
1597: H(X_i\mid Y_i(\cp_i),Y_{\sim i})\le\\
1598: H(X_i\mid Y_i(\cp_i'), \Phi_i)-
1599: H(X_i\mid Y_i(\cp_i), \Phi_i)\, .
1600: %
1601: \end{align*}
1602: %
1603: Here, $Y_i(\cp_i)$ and $Y_i(\cp_i')$ are the result of transmitting
1604: $X_i$ through the channels with parameter $\cp_i$ and $\cp'_i$, respectively.
1605: We claim that
1606: \begin{align*}
1607: & X \rightarrow Y_i(\cp_i) \rightarrow Y_i(\cp_i'), \\
1608: & X \rightarrow Y_{\sim i} \rightarrow \Phi_i, \\
1609: & (Y_i(\cp_i), Y_i(\cp_i')) \rightarrow X \rightarrow (Y_{\sim i}, \Phi_i).
1610: \end{align*}
1611: The first claim follows from the assumption
1612: that the channel family is degraded
1613: and the second claim is also part of the assumption.
1614: Finally, the third claim is true since
1615: the channel is memoryless.
1616:
1617: The thesis is therefore a consequence of
1618: Lemma \ref{lem:infotheoretic} stated below by making
1619: the following substitutions:
1620: \begin{xalignat*}{3}
1621: Y_i(\cp_i) & \rightarrow Y, &
1622: Y_i(\cp_i') & \rightarrow Y', \\
1623: Y_{\sim i} & \rightarrow Z, \;\;&
1624: \Phi_i & \rightarrow Z'.
1625: \end{xalignat*}
1626: %
1627: \end{proof}
1628: %
1629: \begin{lemma}
1630: \label{lem:infotheoretic}
1631: %
1632: Assume that $X\to Y \to Y'$, $X\to Z\to Z'$, as well as
1633: $(Y,Y') \to X \to (Z,Z')$ form Markov chains.
1634: Then
1635: %
1636: \begin{align}
1637: %
1638: H(X\mid Y',Z) - H(X \mid Y,Z) \le H(X\mid Y',Z') - H(X\mid Y,Z')\, .
1639: %
1640: \end{align}
1641: %
1642: \end{lemma}
1643: %
1644: \begin{proof}
1645: %
1646: The statement is equivalent to
1647: $H(X\mid Z,Y',Z') - H(X \mid Y,Z,Y',Z') \le H(X\mid Y',Z') -
1648: H(X\mid Y,Y',Z')$. Let us now condition on a event
1649: $(Y'=y',Z'=z')$. The proof is completed by showing that
1650: (here the conditioning upon $Y'=y',Z'=z'$ is left implicit for the
1651: sake of simplicity)
1652: %
1653: \begin{eqnarray}
1654: %
1655: H(X\mid Y,Z)-H(X\mid Y)-H(X\mid Z)+ H(X)\ge 0\, .
1656: %
1657: \end{eqnarray}
1658: %
1659: This inequality can be written in terms of mutual information
1660: as $I(Y;X\mid Z)\le I(Y;X)$. The statement is therefore a well-known
1661: consequence of the data processing inequality, see~\cite[p. 33]{CoT91},
1662: if we can show that, conditioned on $Y'=y',Z'=z'$,
1663: $Y\to X\to Z$ forms a Markov chain. In formulae, we have to show that
1664: $p(y, z \mid x, y', z') = p(y \mid x, y', z') p(z \mid x, y', z')$,
1665: which in turn follows if we can show that
1666: $\frac{p(z \mid x, y', z')}{p(z \mid x, y, y', z')}=1$.
1667: The last equality can be shown by first applying Bayes law,
1668: then expanding all terms in the order $x, z', y$ and $y'$,
1669: further canceling common terms and, finally,
1670: repeatedly using the conditions
1671: that $X\to Y \to Y'$, $X\to Z\to Z'$, as well as
1672: $(Y,Y') \to X \to (Z,Z')$ form Markov chains.
1673: %
1674: \end{proof}
1675:
1676: In case of linear codes, and communication over a smooth and degraded family of
1677: $\BMS$ channels,
1678: Lemma \ref{lem:gexitlinear} provides an explicit representation of the
1679: $\gexit$ function in terms of $L$-densities. In this case Lemma
1680: \ref{prop:physicaldegradationgeneral}
1681: becomes a statement on the corresponding
1682: kernel. For completeness, let us state the corresponding
1683: condition explicitly.
1684: %
1685: \bcor[$ \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z$ Preserves Partial Order]
1686: \label{lemma:orderingviaphysicaldegradationexit}
1687: Consider a smooth and degraded family of \BMS\ channels characterized by
1688: the associated family of $L$-densities
1689: $\{\Ldens{c}_{\BMSsmall(\ent)}\}_{\ent}$.
1690: Let $\Ldens{a}$ and $\Ldens{b}$ denote two symmetric $L$-densities
1691: so that $\Ldens{a} \prec \Ldens{b}$, i.e.,
1692: $\Ldens{b}$ is physically degraded with respect to $\Ldens{a}$.
1693: Then
1694: \begin{align*}
1695: \int_{-\infty}^{\infty} \!\Ldens{a}(z) \, \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z \text{d}z \leq
1696: \int_{-\infty}^{\infty} \!\Ldens{b}(z)\, \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z \text{d}z.
1697: \end{align*}
1698: \ecor
1699: An alternative proof of this statement
1700: is provided
1701: in Appendix \ref{sec:AlternativePhysicalDegradation}.
1702:
1703: We continue by examining some limiting cases. In the sequel
1704: $\perr$ denotes the error-probability operator. In the
1705: $L$-domain it is defined as
1706: $\perr (\Ldens{a}) = \frac12
1707: \int_{-\infty}^{\infty} \Ldens{a}(z) e^{-(|z/2|+z/2)} \text{d}z$.
1708: \blemma[Bounds for $\gexit$ Kernel]\label{lem:GeneralBounds}
1709: Let $\gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} z$
1710: be the kernel associated to a smooth degraded family of $\BMS$ channels
1711: characterized by their family of $L$-densities
1712: $\{ {\Ldens{c}_{\BMSsmall(\ent)}} \}_{\ent}$.
1713: Then
1714: \begin{align*}
1715: 1-z \leq \gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} z \leq 1.
1716: \end{align*}
1717: Therefore, if $\Ldens{a}$ is a symmetric $L$-density, we have
1718: \begin{align*}
1719: 2 \perr (\Ldens{a}) \leq
1720: \int_{-\infty}^{\infty} \gexitkl {{\Ldens{c}_{\BMSsmall(\ent)}}} z
1721: \Ldens{a}(z) \text{d}z \leq 1\, .
1722: \end{align*}
1723: \elemma
1724: \begin{proof}
1725: In Appendix \ref{sec:AlternativePhysicalDegradation}, we show that
1726: $\gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} {z}$ is non-increasing and concave.
1727: The upper bound follows from
1728: $\gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} z<
1729: \gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} {z=0}=1$.
1730: The lower bound is proved in a similar way by using concavity
1731: and observing that
1732: $\gexitkabsd {\Ldens{c}_{\BMSsmall(\ent)}} {z=1}=0$.
1733: The final claim now follows
1734: from the fact that the $|D|$-domain kernel associated to $\perr$
1735: is equal to $(1-z)/2$.
1736: \end{proof}
1737:
1738: \blemma[Further Properties of $\gexit$ Functions]
1739: Let $\gexitf {}(\ent)$ be the $\gexit$ function associated
1740: to a proper binary linear code of minimum distance larger than $1$,
1741: and transmission over a complete smooth family of $\BMS$ channels. Then
1742: \begin{xalignat*}{3}
1743: \gexitf {}(0) & = 0, & \gexitf {}(1) & = 1.
1744: \end{xalignat*}
1745: If the minimum distance of the code is larger than $k$, then
1746: \begin{align*}
1747: \left.
1748: \frac{\text{d}^{k-1} \phantom{\ent}}{\text{d} \ent^{k-1}}
1749: \gexitf {}(\ent)
1750: \right|_{\ent=0} = 0.
1751: \end{align*}
1752: Further, $\gexitf {}(\ent)$ is a non-decreasing function in $\ent$.
1753: \elemma
1754: \begin{proof}
1755: Consider the first two assertions. If $\ent=0$,
1756: then the associated $L$-density corresponds to a ``delta at infinity''
1757: (this is an easy consequence of the minimum distance being at least
1758: $2$). On the other
1759: hand, if $\ent=1$ then the corresponding $L$-density is a ``delta at zero.''
1760: The claim
1761: in both cases follows now by a direct calculation.
1762:
1763: In order to prove the last claim, we use the definition of
1764: $\gexitf {}(\ent)$ to write
1765: \begin{align*}
1766: \left.
1767: \frac{\text{d}^{k-1} \phantom{\ent}}{\text{d} \ent^{k-1}} \gexitf {}(\ent)
1768: \right|_{\ent=0} =
1769: \frac{1}{n}\left.\frac{\text{d}^{k} \phantom{\ent}}{\text{d} \ent^{k}}
1770: H(X|Y(\ent))
1771: \right|_{\ent=0}\, .
1772: \end{align*}
1773: In order to evaluate the last derivative, we can first assume
1774: that the $i$-th bit is transmitted through a channel $\BMS(\ent_i)$.
1775: Next we take partial derivatives with respect to $k$ of the entropies
1776: $\{\ent_i\}$. Finally we set $\ent_i=0$ for all bits $i$.
1777: We get therefore (neglecting the factor $1/n$):
1778: \begin{align*}
1779: \left.\sum_{i_1\dots i_{k}}
1780: \frac{\partial^{k} \phantom{\ent}}{\partial \ent_{i_1}\cdots
1781: \partial \ent_{i_k}}
1782: H(X|Y)\right|_{\ent_i=0}.
1783: \end{align*}
1784: Of course $h_i$ can be set to $0$ right at the beginning for all
1785: the bits that are not differentiated over. This is equivalent
1786: to passing the exact bits $X_i$. We get the expression
1787: \begin{align*}
1788: \sum_{i_1\dots i_{k}}
1789: \frac{\partial^{k} \phantom{\ent}}{\partial \ent_{i_1}\cdots
1790: \partial \ent_{i_k}}
1791: H(X|Y_{i_1}(\ent_{i_1})\dots Y_{i_k}(\ent_{i_{k}}), X_{\sim i_1\dots i_k} )
1792: \end{align*}
1793: to be evaluated at $\ent_{i_1}=\cdots=\ent_{i_k}=0$. If the
1794: code has minimum distance larger than $k$, then any $n-k$ bits determine the
1795: whole codeword and
1796: $H(X|Y_{i_1}(\ent_{i_1})\dots Y_{i_k}(\ent_{i_{k}}),
1797: X_{\sim i_1\dots i_k} )=0$. This finishes the proof.
1798: \end{proof}
1799:
1800: So far we have used the compact notation $\gexitf {}(\ent)$
1801: for the GEXIT function. In some circumstance it is more convenient to
1802: use a notation that makes the dependence of the functional on the
1803: involved densities more explicit.
1804: \bdefi[Alternative Notation for GEXIT Functional]
1805: Consider a binary linear code and transmission over a smooth
1806: family of BMS channels characterized by the associated family of $L$-densities
1807: $\{\Ldens{c}_{\cp}\}_{\cp}$.
1808: Let $\{\Ldens{a}_{\cp}\}_{\cp}$ denote the associated family of average extrinsic
1809: MAP densities (which we assume smooth).
1810: Define
1811: \begin{align*}
1812: \gentropy(\Ldens{c}_{\cp}, \Ldens{a}_{\cp}) & \defas
1813: \int_{-\infty}^{\infty} \Ldens{a}_{\cp}(z) \gexitkl {\Ldens{c}_{\cp}} z \text{d}z,
1814: \end{align*}
1815: where
1816: \begin{align*}
1817: \gexitkl {\Ldens{c}_{\cp}} z
1818: & =
1819: \frac{\int_{-\infty}^{\infty} \frac{\text{d} \Ldens{c}_{\cp}(w)}{\text{d} \cp}
1820: \lognat(1+e^{-z-w}) \text{d}w}{
1821: \int_{-\infty}^{\infty} \frac{\text{d} \Ldens{c}_{\cp}(w)}{\text{d} \cp}
1822: \lognat(1+e^{-w}) \text{d}w}.
1823: \end{align*}
1824: \edefi
1825: \blemma[GEXIT and Dual GEXIT Function]
1826: \label{lem:dualgexit}
1827: Consider a binary code $C$ and transmission over a complete and smooth
1828: family of BMS channels characterized by the associated family of $L$-densities
1829: $\{\Ldens{c}_\cp\}_{\cp}$. Let $\{\Ldens{a}_{\cp}\}_{\cp}$ denote
1830: the corresponding family of (average) extrinsic \MAP\ densities.
1831: Then the standard GEXIT curve is given in parametric form by
1832: $\{\entropy(\Ldens{c}_{\cp}), \gentropy(\Ldens{c}_{\cp}, \Ldens{a}_{\cp})\}$.
1833: The {\em dual}
1834: GEXIT curve is defined by
1835: $\{\gentropy(\Ldens{a}_{\cp}, \Ldens{c}_{\cp}), \entropy(\Ldens{a}_{\cp})\}$.
1836: Both, standard and dual GEXIT curve have an area equal to
1837: $r(C)$, the rate of the code.
1838: \elemma
1839: Discussion:
1840: Note that both curves are ``comparable'' in that the first component measures
1841: the channel $\Ldens{c}$ and the second argument measure the \MAP\ density
1842: $\Ldens{a}$. The difference between the two
1843: lies in the choice of measure which is applied to each component.
1844:
1845: \begin{proof}
1846: The statement that $\{\entropy(\Ldens{c}_{\cp}),
1847: \gentropy(\Ldens{c}_{\cp}, \Ldens{a}_{\cp})\}$
1848: represents the standard GEXIT function follows by unwinding the
1849: corresponding definitions.
1850: The only statement that requires a proof is the one concerning the
1851: area under the ``dual GEXIT'' curve. We proceed as follows:
1852: Consider the entropy
1853: $\entropy(\Ldens{c}_{\cp} \conv \Ldens{a}_{\cp})$. We have
1854: \begin{align*}
1855: \entropy(\Ldens{c}_{\cp} \conv \Ldens{a}_{\cp}) & =
1856: \int_{-\infty}^{\infty} \Bigl(\int_{-\infty}^{\infty}
1857: \Ldens{c}_{\cp}(w) \Ldens{a}_{\cp}(v-w) \text{d}w \Bigr) \log(1+e^{-v}) \text{d}v \\
1858: & = \int_{-\infty}^{\infty} \int_{-\infty}^{\infty}
1859: \Ldens{c}_{\cp}(w) \Ldens{a}_{\cp}(z) \log(1+e^{-w-z}) \text{d}w \text{d}z.
1860: \end{align*}
1861: Consider now $\frac{\text{d} \entropy(\Ldens{c}_{\cp} \conv \Ldens{a}_{\cp})}{\text{d} \cp}$.
1862: Using the previous representation we get
1863: \begin{align*}
1864: \frac{\text{d} \entropy(\Ldens{c}_{\cp} \conv \Ldens{a}_{\cp})}{\text{d} \cp} & =
1865: \int_{-\infty}^{\infty} \int_{-\infty}^{\infty}
1866: \frac{\text{d}\Ldens{c}_{\cp}(w)}{\text{d} \cp} \Ldens{a}_{\cp}(z) \log(1+e^{-w-z}) \text{d}w \text{d}z + \\
1867: & \phantom{=} \int_{-\infty}^{\infty} \int_{-\infty}^{\infty}
1868: \Ldens{c}_{\cp}(w) \frac{\text{d} \Ldens{a}_{\cp}(z)}{\text{d} \cp} \log(1+e^{-w-z}) \text{d}w \text{d}z.
1869: \end{align*}
1870: The first expression can be identified with the standard GEXIT curve
1871: except that it is parameterized by a generic parameter $\cp$.
1872: The second expression is essentially the same, but the roles of
1873: the two densities are exchanged.
1874: Integrate now this relationship over the whole range of $\cp$ and
1875: assume that this range goes from ``perfect'' (channel) to ``useless''.
1876: The integral on the left clearly equals 1. To perform the integrals
1877: on the right, reparameterize the first expression with respect to
1878: $\ent \defas \int_{\infty}^{\infty} \Ldens{c}_{\cp}(w) \lognat(1+e^{-w}) \text{d} w$
1879: so that the integral is equal to the area under the standard GEXIT curve
1880: given by
1881: $\{\entropy(\Ldens{c}_{\cp}), \gentropy(\Ldens{c}_{\cp}, \Ldens{a}_{\cp})\}$.
1882: In the same manner, reparameterize the second expression by
1883: $\ent \defas \int_{\infty}^{\infty} \Ldens{a}_{\cp}(w) \lognat(1+e^{-w}) \text{d} w$.
1884: Therefore the value of second expression is equal the area
1885: under the curve given by
1886: $\{\entropy(\Ldens{a}_{\cp}), \gentropy(\Ldens{a}_{\cp}, \Ldens{c}_{\cp})\}$.
1887: Since the sum of the two areas equals one and the area under the
1888: standard GEXIT curve equals $r(C)$, it follows that the area under
1889: the second curve equals $1-r(C)$. Finally, note that if we consider the inverse
1890: of the second curve by exchanging the two coordinates, i.e., if we consider the
1891: curve
1892: $\{\gentropy(\Ldens{a}_{\cp}, \Ldens{c}_{\cp}), \entropy(\Ldens{a}_{\cp})\}$,
1893: then the area under this curve is equal to $1-(1-r(C))=r(C)$, as claimed.
1894: \end{proof}
1895: \begin{example}[GEXIT Versus Dual GEXIT]
1896: Fig.~\ref{fig:gexitanddualgexit} shows the standard
1897: GEXIT function and the dual GEXIT function for the $[5, 4, 2]$ code
1898: and transmission over the $\BSC$. Although the two curves have quite
1899: distinct shapes, the area under the two curves is the same.
1900: \begin{figure}[htp]
1901: \setlength{\unitlength}{0.75bp}%
1902: \begin{center}
1903: \begin{picture}(120,120)
1904: \put(0,0)
1905: {
1906: \put(0,0){\includegraphics[scale=0.75]{dualgexitbsc3}}
1907: \put(60, -2){\makebox(0,0)[t]{\small $\entropy(\Ldens{c}_{\ent})$, $\gentropy(\Ldens{a}_{\ent}, \Ldens{c}_{\ent})$}}
1908: \put(-2, 60){\makebox(0,0)[r]{\small \rotatebox{90}{$\gentropy(\Ldens{c}_{\ent}, \Ldens{a}_{\ent})$,$\entropy(\Ldens{a}_{\ent})$}}}
1909: \put(60, 30){\makebox(0,0)[t]{\small both GEXIT}}
1910: \put(-2, -2){\makebox(0,0)[rt]{\small $0$}}
1911: \put(120,-2){\makebox(0,0)[t]{\small $1$}}
1912: \put(-2,120){\makebox(0,0)[r]{\small $1$}}
1913: }
1914: \end{picture}
1915: \end{center}
1916: \caption{\label{fig:gexitanddualgexit} Standard and dual GEXIT function of $[5, 4, 2]$
1917: code and transmission over the $\BSC$.}
1918: \end{figure}
1919: \end{example}
1920:
1921: %
1922: %************************************************************************
1923: %
1924: \section{Ensembles: Concentration and Asymptotic Setting}
1925: \label{sec:asymptotic}
1926: For simple codes, like, e.g.,
1927: single parity-check codes or repetition codes, $\exitf {}$
1928: and $\gexitf {}$ are relatively easy to compute.
1929: In general though it is not a trivial matter to
1930: determine the density of $\Phi^{}_i$ required
1931: for the calculation.
1932: What we {\em can} typically compute are the extrinsic estimates
1933: if we use the $\BP$ decoder instead of the $\MAP$ decoder. It is therefore
1934: natural to look at the equivalent of $\exit$ and $\gexit$ functions
1935: if we substitute the extrinsic $\MAP$ estimates by their equivalent extrinsic
1936: $\BP$ estimates.
1937: Although most of the subsequent definitions and statements
1938: can be as easily derived for $\exit$ as for $\gexit$ functions,
1939: we
1940: focus on the latter. After all, these are the natural objects
1941: to study as suggested by the GAT.
1942: \bdefi[$\gexitf {\BPsmall}$ for Linear Codes and $\BMS$ Channels]
1943: \label{def:bpgexit}
1944: Let $X$ be chosen uniformly at random from a proper binary linear code.
1945: Let the channel from $X$ to $Y$ be memoryless, where
1946: $Y_i$ is the result of passing $X_i$ through the
1947: smooth family $\{\BMS(\ent_i)\}_{\ent_i}$, $\ent_i \in [0, 1]$.
1948: Assume that all individual channels are parameterized in a smooth way
1949: by a common parameter $\cp$, i.e., $\ent_i=\ent_i(\cp)$, $i \in [n]$.
1950: Let $\Phi^{\BPsmall, \itersmall}_i$ denote the extrinsic estimate of the
1951: $i^{\text{th}}$ bit at the $\iter^{\text{th}}$ round of $\BP$ decoding, assuming
1952: an arbitrary but fixed representation of the code by a Tanner graph
1953: as well as an arbitrary but fixed schedule of the decoder.
1954: Then the $\BP$ $\gexit$ function is defined as
1955: \begin{align*}
1956: \gexitfi {\BPsmall, \itersmall}(\cp)
1957: & \defas
1958: \frac{\partial H(X_i | \Phi^{\BPsmall, \itersmall}_i, Y_i)}{\partial \ent_i}
1959: \frac{\text{d} \ent_i}{\text{d} \cp} \Big|_{\cp}.
1960: \end{align*}
1961: \edefi
1962: The following statement, which is a direct consequence of the previous
1963: definition and Lemma \ref{prop:physicaldegradationgeneral}, confirms
1964: the intuitive fact that the BP GEXIT function (which is associated
1965: to the suboptimal BP decoder) is at least as large as the the GEXIT
1966: function itself, assuming only that the channel family is degraded.
1967: \bcor[GEXIT Versus BP GEXIT]
1968: \label{cor:gexitversusbpgexit}
1969: Let $X$ be chosen uniformly at random from a proper binary linear code.
1970: Let the channel from $X$ to $Y$ be memoryless, where
1971: $Y_i$ is the result of passing $X_i$ through a
1972: smooth and degraded family $\{\BMS(\ent_i)\}_{\ent_i}$, $\ent_i \in [0, 1]$.
1973: Assume that all individual channels are parameterized in a smooth (differentiable) way
1974: by a common parameter $\cp$, i.e., $\ent_i=\ent_i(\cp)$, $i \in [n]$.
1975: Let $\gexitfi{}(\cp)$ and $\gexitfi {\BPsmall, \itersmall}(\cp)$ be as defined in Definitions \ref{def:gexit} and \ref{def:bpgexit}.
1976: Then
1977: \begin{align*}
1978: \gexitfi{}(\cp) \leq \gexitfi {\BPsmall, \itersmall}(\cp).
1979: \end{align*}
1980: \ecor
1981: \bdefi[Asymptotic $\BP$ $\exit$ and $\gexit$ Functions]
1982: \label{def:asymptoticbpgexit}
1983: Consider a \ddp $(\ledge, \redge)$ and the corresponding sequence of
1984: ensembles $\eldpc n \ledge \redge$. Further consider a smooth
1985: and degraded family $\{\BMS(\ent)\}_{\ent}$.
1986: Assume that all bits of $X$ are
1987: sent through the channel $\BMS(\ent)$.
1988: For $\graph \in \eldpc n \ledge \redge$ and $i \in [n]$, let
1989: $\gexitfi {}(\graph, \cp)$ and
1990: $\gexitfi {\BPsmall, \itersmall}(\graph, \cp)$
1991: denote the $i^{\text{th}}$ $\MAP$ and $\BP$
1992: $\gexit$ function associated to code $\graph$.
1993: By some abuse of notation, define
1994: the asymptotic (and average) quantities
1995: \begin{align*}
1996: \gexitf {}(\ent) & \defas
1997: \limsup_{n \rightarrow \infty} \expectation_{\graph}
1998: \Bigl[\frac{1}{n} \sum_{i \in [n]}\gexitfi {}(\graph, \ent) \Bigr], \\
1999: \gexitf {\BPsmall, \itersmall}(\ent) & \defas
2000: \lim_{n \rightarrow \infty} \expectation_{\graph}
2001: \Bigl[
2002: \frac{1}{n} \sum_{i \in [n]} \gexitfi {\BPsmall, \itersmall}(\graph, \ent)
2003: \Bigr], \\
2004: \gexitf {\BPsmall}(\ent) & \defas
2005: \lim_{\iter \rightarrow \infty} \gexitf {\BPsmall, \itersmall}(\ent).
2006: \end{align*}
2007: For notational simplicity we suppress the dependence of the
2008: above quantities on the \ddp and the channel family $\{\BMS(\ent)\}_{\ent}$.
2009: \edefi
2010:
2011: In the above definitions we have taken the average of the individual curves
2012: over the ensemble. Let us now justify this approach by showing that the quantities
2013: are concentrated. The proof of the following statement,
2014: which asserts the concentration of the conditional entropy,
2015: can be found in \cite{MMU05}.
2016: \btheo[Concentration of Conditional Entropy]
2017: \label{theo:concentrationml}
2018: Let $\graph(n)$ be chosen uniformly at random from $\eldpc n \ledge \redge$.
2019: Assume that $\graph(n)$ is used to transmit over a $\BMS(\ent)$ channel.
2020: By some abuse of notation, let
2021: $H_{\graph(n)} =H_{\graph(n)}(X \mid Y)$ be the associated conditional entropy.
2022: Then for any $\xi > 0$
2023: %
2024: \begin{align*}
2025: \Pr \left\{|H_{\graph(n)} - \expectation_{\graph(n)}[H_{\graph(n)}]|>\n \xi\right\} &\leq 2\,
2026: \text{e}^{-\n B \xi^2},\\
2027: \end{align*}
2028: where $B= 1/(2 (\drmax+1)^2(1-r))$ and $\drmax$ is the maximal check-node degree.
2029: \etheo
2030:
2031: \btheo[Concentration of $\gexitf {\BPsmall, \iter}$]
2032: \label{theo:concentrationexitbp}
2033: Consider the sequence of ensembles
2034: $\eldpc n \ledge \redge$, where $(\ledge, \redge)$ is fixed
2035: and $n$ tends to infinity. Then the limits
2036: $\gexitf {\BPsmall, \itersmall}(\ent) =
2037: \lim_{n \rightarrow \infty} n^{-1}\expectation_{\graph}[
2038: \sum_{i \in [n]} \gexitfi {\BPsmall, \itersmall}(\graph, \ent)]$ and
2039: $\gexitf {\BPsmall}(\ent) =
2040: \lim_{\iter \rightarrow \infty} \gexitf {\BPsmall, \itersmall}(\ent)$
2041: exist.
2042: Further, let $\graph(n)$ be chosen uniformly at random from
2043: $\eldpc n \ledge \redge$.
2044: Assume that $\graph(n)$ is used to transmit over a $\BMS(\ent)$ channel.
2045: Then, for all $\xi>0$, there exists $\alpha_\xi>0$, such that,
2046: for $n$ large enough
2047: \begin{multline}
2048: \Pr\Bigl\{\Bigl| \gexitf {\BPsmall, \itersmall}(\graph(n), \ent)-
2049: \gexitf {\BPsmall, \itersmall}(\ent)
2050: \Bigr|> n \xi\Bigr\}
2051: \leq \text{e}^{-\alpha_\xi \n}.
2052: \end{multline}
2053: \etheo
2054: \begin{proof}
2055: Note that for a fixed iteration number $\iter$, the distribution
2056: of $\Phi^{\BPsmall}_i$ (with $i$ a uniformly random node), assuming
2057: that the all-one codeword was sent, converges (at a speed of $1/n$)
2058: to the corresponding
2059: distribution of density evolution,
2060: denote it by $\Ldens{a}_{\iter}$.
2061: The result now follows by
2062: noting that $\gexitf {\BPsmall, \itersmall}$ is the result of
2063: applying a bounded linear operator to this distribution
2064: $\Ldens{a}_{\iter}$.
2065: The proof of concentration is almost verbatimly the same as
2066: the proof in \cite{RiU05},
2067: which shows the concentration of the probability
2068: of error under $\BP$ decoding, or the proof in \cite{MMU05},
2069: which relates to the
2070: concentration of the $\BP$ $\exit$ function. We will therefore skip the details.
2071: \end{proof}
2072: \btheo[Concentration of $\gexitf {}$]
2073: \label{theo:concentrationexitml}
2074: Let $\graph$ be chosen uniformly at random from $\ldpc(\n,\ledge,\redge)$
2075: and consider the smooth and degraded family $\{\BMS(\ent)\}_{\ent}$, $\ent \in [0, 1]$.
2076: Assume that $\graph$ is used to transmit over the $\BMS(\ent)$ channel.
2077: Let $H_{\graph(n)} =H_{\graph(n)}(X \mid Y)$ be the associated conditional entropy, $\gexitf {}(\graph(n),\ent)$ the corresponding $\MAP$ $\gexit$ function,
2078: and $\gexitf {}_n(\ent)=\expectation\,\gexitf {}(\graph(n),\ent)$.
2079: Let $J \subseteq [0, 1]$ be an interval on which
2080: $\lim_{n\to\infty} \frac{1}{n}\expectation \left[H_{\graph(n)}\right]$
2081: exists and is differentiable with respect
2082: to $\ent$. Then, for any $\cp\in J$ and $\xi>0$ there exist an
2083: $\alpha_{\xi}>0$ such that, for $n$ large enough
2084: \begin{align*}
2085: \Pr \left\{|\gexitf {}(\graph(n),\ent)-\gexitf {}_n(\ent)|>\n \xi\right\} &\leq
2086: \text{e}^{-\n \alpha_{\xi}}.
2087: \end{align*}
2088: Furthermore, if
2089: $\lim_{n\to\infty} \frac{1}{n}\expectation \left[H_{\graph(n)}\right]$
2090: is twice differentiable with respect
2091: to $\ent \in J$, there exists a
2092: strictly positive constant $A$ such that $\alpha_\xi>A\xi^4$.
2093: \etheo
2094: %
2095: The proof of this statement can be found in \cite{MMU05}.
2096:
2097: Let us summarize. We have seen that all the quantities which we introduced
2098: in Definition \ref{def:asymptoticbpgexit} are concentrated
2099: and that the $\BP$
2100: quantities $\gexitf {\BPsmall, \itersmall}$ and $\gexitf {\BPsmall}$ exist.
2101: Unfortunately, we had to use $\limsup$ for the definition of
2102: $\gexitf {}$ since to prove the existence of the limit seems
2103: to be difficult.
2104: As discussed in \cite{MMU05}, even in the case of transmission
2105: over the $\BEC$
2106: the existence of the corresponding limit is not known in general but only follows
2107: from the explicit construction of the Maxwell decoder in all those cases where
2108: the Maxwell construction can be shown to result in $\MAP$ performance.
2109:
2110: Note that $\gexitf {\BPsmall, \itersmall}$ and $\gexitf {\BPsmall}$
2111: have again a convenient representation in terms of the asymptotic $\BP$ densities.
2112: More precisely, we have
2113: \begin{align*}
2114: \gexitf {\BPsmall, \itersmall} (\ent) & =
2115: \int_{-\infty}^{\infty} \Ldens{a}^{\BPsmall, \itersmall} (z)
2116: \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z
2117: \;\text{d}z, \\
2118: \gexitf {\BPsmall} (\ent) & =
2119: \int_{-\infty}^{\infty} \Ldens{a}^{\BPsmall} (z)
2120: \gexitkl {\Ldens{c}_{\BMSsmall(\ent)}} z
2121: \;\text{d}z,
2122: \end{align*}
2123: where $\Ldens{a}^{\BPsmall, \itersmall}$ is the limiting density
2124: of $\Phi_i^{\BPsmall, \iter}$ (with $i$ a uniformly random node) under the
2125: all-one codeword assumption as $n$ tends to infinity
2126: associated to the \ddp $(\ledge, \redge)$. This density can easily be computed
2127: by density evolution. In a similar manner, $\Ldens{a}^{\BPsmall}$ is the
2128: corresponding fixed-point density of density evolution.
2129:
2130: In Fig.~\ref{fig:gexitbsc} we plot the $\BP$ $\gexit$ function
2131: $\gexitf {\BPsmall}$ for a
2132: few regular LDPC ensembles
2133: and we compare them
2134: with the corresponding $\BP$ $\exit$ functions,
2135: which we denote by $\exitf {\BPsmall}$. We see
2136: that the curves are quite similar.
2137: \begin{figure}[hbt]
2138: \centering
2139: \setlength{\unitlength}{0.6bp}
2140: \begin{picture}(400,200)
2141: \put(0,0)
2142: {
2143: \put(10,0){\includegraphics[scale=0.6]{exitcollectionbsc}} %length=180bp
2144: {\tiny
2145: %\footnotesize
2146: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2147: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2148: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2149: }
2150: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2151: %
2152: \put(130,25){\makebox(0,0)[lb]{\footnotesize{$(3, 4)$}}}
2153: \put(90,25){\makebox(0,0)[lb]{\footnotesize{$(3, 6)$}}}
2154: \put(85,90){\makebox(0,0)[rb]{\footnotesize{$(4, 8)$}}}
2155: \put(52,35){\makebox(0,0)[lb]{\footnotesize{$(2, 4)$}}}
2156: \put(45,25){\makebox(0,0)[rb]{\footnotesize{$(2, 6)$}}}
2157: }
2158: \put(200,0)
2159: {
2160: \put(10,0){\includegraphics[scale=0.6]{exitcollectionbawgnc}} %length=180bp
2161: {\tiny
2162: %\footnotesize
2163: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2164: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2165: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2166: }
2167: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2168: %
2169: \put(130,25){\makebox(0,0)[lb]{\footnotesize{$(3, 4)$}}}
2170: \put(90,25){\makebox(0,0)[lb]{\footnotesize{$(3, 6)$}}}
2171: \put(85,90){\makebox(0,0)[rb]{\footnotesize{$(4, 8)$}}}
2172: \put(52,35){\makebox(0,0)[lb]{\footnotesize{$(2, 4)$}}}
2173: \put(45,25){\makebox(0,0)[rb]{\footnotesize{$(2, 6)$}}}
2174: }
2175: \end{picture}
2176: \newcaption{$\BP$ $\gexit$ (solid curves)
2177: versus $\BP$ $\exit$ (dashed curves) for several
2178: regular LDPC ensembles for the $\BSC$ (left picture) and the
2179: $\BAWGNC$ (right picture).}
2180: \label{fig:gexitbsc}
2181: \end{figure}
2182:
2183: \blemma[$\gexitf {} \leq \gexitf {\BPsmall}$]
2184: \label{lemma:gexitmapversusgexitbp}
2185: Consider a \ddp $(\ledge, \redge)$ and transmission over the
2186: smooth and degraded family
2187: $\{\BMS(\ent)\}_{\ent}$.
2188: Let $\gexitf {}(\ent)$ and $\gexitf {\BPsmall}(\ent)$ denote
2189: respectively the corresponding asymptotic $\MAP$ and $\BP$ $\gexit$ functions
2190: as defined in Definition \ref{def:asymptoticbpgexit}
2191: when the code is chosen uniformly at random from the ensemble
2192: $\eldpc n \ledge \redge$.
2193: Then
2194: %
2195: \begin{align*}
2196: %
2197: \gexitf {}(\ent) \leq \gexitf {\BPsmall} (\ent).
2198: % \leq \gexitf {\BPsmall, \iter} (\ent)\, .
2199: %
2200: \end{align*}
2201: \elemma
2202: \begin{proof}
2203: Using Corollary \ref{cor:gexitversusbpgexit}, we know that for any
2204: $\graph \in \eldpc n \ledge \redge$ and $\ell \in \naturals$
2205: \begin{align*}
2206: {\gexitf {}}_{\graph}(\cp) \leq {\gexitf {}}^{\BPsmall, \ell}_{\graph}(\cp).
2207: \end{align*}
2208: If we take first the expectation over the elements of the ensemble,
2209: then the $\limsup$ on both sides with respect to $n$, and finally
2210: the limit $\ell \rightarrow \infty$, we get the desired result.
2211:
2212: \end{proof}
2213:
2214: \section{An Upper Bound on the MAP Threshold}
2215: \label{sec:upperbound}
2216: One important consequence of the area theorem is that it gives
2217: rise to an easy to compute upper bound on the threshold
2218: of $\MAP$ decoding.
2219: \bdefi[$\MAP$ Threshold]
2220: Consider a \ddp $(\ledge, \redge)$ and a smooth and degraded family
2221: $\{\BMS(\ent)\}_{\ent}$. The {\em threshold} $\ent^{\MAPsmall}$ is {\em defined} as
2222: \begin{align*}
2223: \ent^{\MAPsmall} & \defas
2224: \min\{\ent \in [0, 1]: \liminf_{n \rightarrow \infty}
2225: \expectation_{\graph}[H(X \mid Y(\ent))]/n > 0\}.
2226: \end{align*}
2227: \edefi
2228: %Discussion: Let us consider the operational meaning of the above
2229: %definition. Let $\ent < \ent^{\MAPsmall}$. Then for any $\delta > 0$,
2230: %there exists a finite $n = n(\delta)$ so that
2231: %$\expectation_{\graph}[H(X \mid Y(\ent))]/n < \delta$.
2232: %By Theorem \ref{theo:concentrationml} it follows that if
2233: %transmission takes place over the channel with parameter $\ent$
2234: %then all sufficiently
2235: %long codes (of properly chosen blocklength)
2236: %have a normalized conditional entropy less than any fixed constant.
2237: %Since $\delta$ can be chosen arbitrarily small, this means in
2238: %essence that arbitrarily reliable communications is possible with
2239: %most codes at that channel parameter.
2240: %
2241: %On the other hand, assume that $\ent > \ent^{\MAPsmall}$. In this case
2242: %the average (over the ensemble)
2243: %normalized conditional entropy stays bounded away from
2244: %zero by a strictly positive constant for an infinite sequence of
2245: %blocklengths.
2246: %Again, by Theorem \ref{theo:concentrationml}
2247: %this is not only true for the average over the ensemble but
2248: %for most elements from the ensemble. It follows that
2249: %(at least for those blocklengths) reliable transmission is not possible.
2250:
2251: Discussion: Let us consider the operational meaning of the above
2252: definition. Let $\ent < \ent^{\MAPsmall}$. Then by definition of
2253: the threshold,
2254: there exists a sequence of blocklengths $n_1, n_2, n_3, \cdots,$ so
2255: that the normalized (divided by the blocklength $n$)
2256: {\em average} conditional entropy converges to zero.
2257: By Theorem \ref{theo:concentrationml} it follows that most of the codes
2258: in the corresponding ensembles
2259: have a normalized conditional entropy less than any fixed constant.
2260: %This means that for this sequence of blocklengths and the
2261: %conditional entropy behaves sublinearly for at least some
2262: %elements in the ensemble.
2263: For sufficiently
2264: large blocklengths, a conditional entropy which
2265: grows sublinearly implies
2266: that the receiver can limit the set of hypothesis to a subexponential
2267: list which with high probability contains the correct codeword.
2268: Therefore, in this sense reliable communication is possible.
2269:
2270: On the other hand, assume that $\ent > \ent^{\MAPsmall}$. In this case
2271: the normalized conditional entropy stays bounded away from
2272: zero by a strictly positive constant for all sufficiently
2273: large blocklengths. By Theorem \ref{theo:concentrationml}
2274: this is not only true for the average over the ensemble but
2275: for most elements from the ensemble. It follows that
2276: with most elements from the ensemble
2277: reliable communication is not possible.
2278:
2279: \btheo[Upper Bound on $\MAP$ Threshold]\label{theo:UBMAP}
2280: Consider a \ddp $(\ledge, \redge)$ whose
2281: asymptotic rate converges to the design rate $r(\ledge, \redge)$, see \cite[Lemma 7]{MMU05}.
2282: Assume further that transmission takes place over a smooth and degraded family
2283: $\{\BMS(\ent)\}_{\ent}$. Let $\gexitf {\BPsmall}(\ent)$
2284: denote the associated $\BP$ $\gexit$ function.
2285: Then
2286: %
2287: \begin{align}
2288: %
2289: \liminf_{n \to \infty}
2290: \expectation_{\graph}[H(X \mid Y(\ent))]/n\ge r(\ledge, \redge)-\int_{\ent}^1
2291: \gexitf {\BPsmall}(\ent') \;\text{d}\ent'\, .\label{eq:LowerBoundEntropyGen}
2292: %
2293: \end{align}
2294: %
2295: Furthermore, if
2296: $\overline{\ent}$ denotes the largest positive number so that
2297: \begin{align*}
2298: \int_{\overline{\ent}}^{1} \gexitf {\BPsmall}(\ent) \;\text{d}\ent=r(\ledge,
2299: \redge),
2300: \end{align*}
2301: then $\ent^{\MAPsmall} \leq \overline{\ent}$, where
2302: $\ent^{\MAPsmall}$ denotes the $\MAP$ threshold.
2303: \etheo
2304: \begin{proof}
2305: Let $\graph$ be chosen uniformly at random
2306: from the ensemble $\eldpc n \ledge \redge$. By the GAT
2307: %
2308: \begin{align*}
2309: %
2310: r(\ledge, \redge)-& \liminf_{n \to \infty}
2311: \expectation_{\graph}[H(X \mid Y(\ent))]/n = \\
2312: &=\limsup_{n\to\infty} \frac{1}{n}\,
2313: \expectation_{\graph}[H(X \mid Y(1))-H(X \mid Y(\ent))]=\\
2314: &=\limsup_{n \to \infty}\expectation_{\graph} \Bigl[\int_{\ent}^1
2315: \gexitf {}(\graph,\ent') \;\text{d}\ent' \Bigr].
2316: %
2317: \end{align*}
2318: %
2319: We can exchange the expectation and the integral by Fubini's theorem:
2320: in fact $\gexitf {}(\graph,\ent')$ is measurable
2321: and $\gexitf {}(\graph,\ent')\in [0,1]$.
2322: We can furthermore exchange the limit and the integral by the
2323: Fatou-Lebesgue lemma.
2324: We get
2325: %
2326: \begin{align*}
2327: %
2328: \liminf_{n \to \infty}
2329: \expectation_{\graph}[H(X \mid Y(\ent))]/n \geq
2330: r(\ledge, \redge)-\! \int_{\ent}^1
2331: \gexitf {}(\ent') \;\text{d}\ent'\, .
2332: %
2333: \end{align*}
2334: %
2335: Equation (\ref{eq:LowerBoundEntropyGen}) is proved by applying Lemma
2336: \ref{lemma:gexitmapversusgexitbp}.
2337:
2338: The upper bound on the $\MAP$ threshold follows from the
2339: observation that the r.h.s. of Eq.~(\ref{eq:LowerBoundEntropyGen})
2340: is non-decreasing in $h$. Therefore $\limsup_{n \to \infty}
2341: \expectation_{\graph}[H(X \mid Y(\ent))]/n$ is bounded away from $0$
2342: for any $\ent>\overline{\ent}$ and the thesis follows from the definition
2343: of $\ent^{\MAPsmall}$.
2344: \end{proof}
2345:
2346: \bex
2347: The following table presents the upper bounds on the $\MAP$ threshold
2348: for transmission over the $\BAWGNC(\ent)$
2349: as derived from Theorem \ref{theo:UBMAP} for a few regular ensembles:
2350: $\ledge(x) = x^{\ldegree-1}$, $\redge(x) = x^{\rdegree-1}$.
2351: The same threshold were first computed using the (non-rigorous)
2352: replica method from
2353: statistical physics~\cite{andrea}. In~\cite{Mon04}, they were shown to
2354: be upper bounds for $\rdegree$ even, using an interpolation technique.
2355: The present proof applies also to the case of odd $\rdegree$.
2356: It can be proved that the three characterizations of the threshold are
2357: indeed equivalent, i.e., they give {\em exactly} the same value.
2358: \tablespace
2359: \begin{center}
2360: \small{
2361: \begin{tabular}{cccccc}
2362: $\ldegree$ & $\rdegree$ & $\ih^\BP$ & $\overline{\ih}$ & $\overline{\ih} ($\cite{WiS05,BKLM02}) & $\ih^\Sh$ \\ \hline
2363: $3$ & $4$ & $0.6507(5)$ & $0.7417(1)$ & $0.743231$ & $3/4$ \\
2364: $3$ & $5$ & $0.5113(5)$ & $0.5800(3)$ & $0.583578$ & $3/5$ \\
2365: $3$ & $6$ & $0.4160(5)$ & $0.4721(5)$ & $0.476728$ & $1/2$ \\
2366: $4$ & $6$ & $0.5203(5)$ & $0.6636(2)$ & $0.663679$ & $1/3$ \\
2367: \end{tabular}
2368: }
2369: \end{center}
2370: \tablespace
2371: Also shown is the result of the information theoretic upper bound given in
2372: \cite{WiS05}, which in turn is an improved version of the bound
2373: developed in \cite{BKLM02}. For the specific case of transmission over the \BSC\ and
2374: regular codes it is given by $h_2(\overline{\cp})$, where
2375: $\overline{\cp}$ is the unique positive root
2376: of the equation
2377: $\rdegree h_2(\cp)= \ldegree h_2((1-(1-2\cp)^\rdegree)/2)$.
2378: \eex
2379: %
2380: %*****************************************************************
2381: %
2382: \section{The Extended BP $\gexit$ Curve}
2383: \label{sec:egexit}
2384: \subsection{Extended $\BP$ $\gexit$ Curve}
2385: As discussed in detail in \cite{MMU05} for
2386: the case of transmission over the $\BEC$, the fundamental relationship
2387: which appears in the limit of large blocklengths between the $\MAP$
2388: and the $\BP$ decoder is best described in terms of the {\em extended}
2389: $\exit$ curve. For the $\BEC$ this is the curve with parametric
2390: description
2391: $\left( \frac{\xl}{\ledge(1-\redge(1-\xl))}, \lnode(1-\redge(1-\xl)) \right)$,
2392: where $\xl$ takes values in the subset $J\subseteq [0, 1]$
2393: such that $\xl\le\ledge(1-\redge(1-\xl))$
2394: ($J$ is in fact the union of a finite number of intervals).
2395: Note that the families
2396: $\{\Ldens{f}_\xl\}_\xl \defas \{\BEC(\xl)\}_\xl$ and
2397: $\{\Ldens{c}_\xl\}_\xl \defas \{\BEC(\frac{\xl}{\ledge(1-\redge(1-\xl)))}\}_\xl$,
2398: $\xl \in J$, have the following property: For each $\xl \in J$,
2399: $\Ldens{f}_\xl$ constitutes a fixed-point density (of density evolution)
2400: for the channel
2401: $\Ldens{c}_\xl$. Furthermore both channel families are {\em smooth}
2402: and satisfy $H(\Ldens{f}_\xl)=\xl$.
2403: Finally if $J=[0,1]$ (a necessary condition for this to happen is
2404: $\ledge'(0)\redge'(1)\ge 1$)
2405: the families are said to be {\em complete}.
2406: \bdefi[Complete Fixed-Point Family, $\gexitf \EBPsmall$ and $\gexitf \BPsmall$]
2407: Consider a degree distribution pair $(\ledge, \redge)$.
2408: We say that the families $\{\Ldens{f}_\xl\}_\xl$
2409: and $\{\Ldens{c}_\xl\}_\xl$, $\xl \in [0, 1]$, form a {\em complete fixed-point family}
2410: for $(\ledge, \redge)$ if
2411: \begin{itemize}
2412: \item[(i)] there exists a complete and degraded family $\{\BMS(\ent)\}_{\ent}$
2413: such that for each $\xl \in [0, 1]$, $\Ldens{c}_\xl \in \{\BMS(\ent)\}_{\ent}$
2414: \item[(ii)] for each $\xl \in [0, 1]$, $\Ldens{f}_\xl$ is a fixed-point density
2415: with respect to the degree distribution $(\ledge, \redge)$ and the
2416: channel $\Ldens{c}_\xl$; this means that for each $\xl \in[0, 1]$,
2417: $\Ldens{f}_\xl =\Ldens{c}_\xl \conv \ledge(\redge(\Ldens{f}_x))$
2418: \item[(iii)] $\{\Ldens{f}_\xl\}_\xl$ and $\{\Ldens{c}_\xl\}_\xl$ are
2419: smooth with respect to $\xl$
2420: \item[(iv)] $\entropy(\Ldens{f}_\xl)=\xl$
2421: \end{itemize}
2422: Let $\Ldens{a}_\xl(y) \defas \lnode(\redge(\Ldens{f}_\xl))$.
2423: The {\em extended} $\BP$ ($\EBP$) $\gexit$ curve,
2424: call it $\gexitf {\EBPsmall}(\xl)$, is then given in parametric form
2425: by $(H(\xl), \gexitf {\EBPsmall}(\xl))$, where
2426: \begin{align*}
2427: \gexitf {\EBPsmall}(\xl) & \defas
2428: \int_{-\infty}^{\infty} \Ldens{a}_\xl(y) \gexitkl {\Ldens{c}_\xl} y \;\text{d}y.
2429: \end{align*}
2430: Finally, the $\BP$ $\gexit$ curve, call it $\gexitf \BPsmall$,
2431: is the ``envelope''
2432: of the $\gexitf {\EBPsmall}$ curve.
2433: \edefi
2434: Discussion: Contrary to our usual notation, we have used $\xl$ to parameterize
2435: the channel families and the function $\gexitf {\EBPsmall}(\xl)$
2436: and we have assumed that $\entropy(\Ldens{f}_\xl)=\xl$ (rather
2437: than $\entropy(\Ldens{c}_\xl)=\xl$). This has the following reason: in general,
2438: the $\EBP$ $\gexit$ function is not a single-valued function of the
2439: {\em channel} entropy but it is a single-valued function of the
2440: {\em fixed-point} entropy (see Fig.~\ref{fig:multijump}). We prefer
2441: to use the parameter $\xl$ instead of the usual parameter $\ent$,
2442: to remind ourselves that the channel $\Ldens{c}_{\xl}$ is the
2443: channel which belongs to the fixed-point density $\Ldens{f}_{\xl}$
2444: (and not the channel $\Ldens{c}_{\ent}$, which by our previous
2445: notational convention has entropy $\ent$). {\em Complete} fixed-point
2446: families do not always exist.
2447: If, for instance, $\ledge_2=0$,
2448: then $\xl$ cannot be chosen arbitrarily close to $0$.
2449: This is easily seen for transmission over the $\BEC$.
2450: In this case $\xl\ge \underline{\xl}$ with $\underline{\xl}$
2451: the smallest (non-vanishing) root of the
2452: equation $\ledge(1-\rho(1-\xl))=\xl$.
2453:
2454: From the definition it is not immediately obvious
2455: that for a given degree distribution pair $(\ledge, \redge)$
2456: and a complete and degraded family $\{\BMS(\ent)\}_{\ent}$,
2457: such a (complete or incomplete) fixed-point family always exists,
2458: or that it is unique.
2459: For the $\BEC$ we have an explicit formula for the family, but in the general
2460: case the existence is far from trivial.
2461: We will get back to this point in the next section.
2462:
2463: One of the important applications of the $\EBP$ $\gexit$ curve is that
2464: it encodes very clearly the connection between $\MAP$ and $\BP$ decoding.
2465: As mentioned above, the $\BP$ $\gexit$ function is obtained
2466: as the `envelope' of the $\EBP$ curve. More precisely,
2467: one has to choose, for each value of the channel entropy $\ent$,
2468: the branch of the $\EBP$ curve whose $\gexit$ value is the largest.
2469: As pointed out in the introduction when discussing Fig.~\ref{fig:multijump},
2470: a different single valued function can be obtained by applying the Maxwell
2471: construction, described in detail in \cite{MMU05}, to the $\EBP$ $\gexit$
2472: curve. Motivated by the GAT as well as by the $\BEC$ case, we formulate
2473: the following
2474: %
2475: \begin{conj}
2476: \label{con:theconjecture}
2477: The ($\MAP$)
2478: $\gexit$ function $\gexitf{}(\ent)$ is obtained by applying the Maxwell
2479: construction to the extended $\BP$ $\gexit$ curve $(H(\xl),
2480: \gexitf {\EBPsmall}(\xl))$.
2481: \end{conj}
2482:
2483: Let us
2484: consider a few typical examples. In each of the following
2485: cases the complete fixed-point family was computed by a {\em numerical}
2486: procedure, which will be explained in the next section.
2487: \bex[LDPC($x,x^5$) -- BSC] \label{ex:CycleEx}
2488: Consider the \ddp $(\ledge,\redge)=(x, x^5)$ and the
2489: corresponding $\ldpc$ ensemble with design rate $\drate=2/3$.
2490: We assume that transmission takes place over the family
2491: $\{\text{BSC}(\cp)\}$. Recall that for this code the
2492: $\BP$ threshold is given by the stability condition.
2493: From Fig.~\ref{fig:26ebpmapgexit} we see that, according to
2494: the numerical calculation, the $\EBP$ $\gexit$ curve is a monotone
2495: function. Assuming this is true, it follows that
2496: the $\EBP$ $\gexit$ is equal to the $\BP$ $\gexit$ curve for
2497: this example.
2498: For any value of the channel parameter a single fixed point
2499: density (apart from the `delta at infinity') is found.
2500: Also: a single fixed point density exists for each value of the
2501: density entropy $\xl$. The Maxwell construction is trivial in this
2502: case and yields a $\MAP~ \gexit$ equal to
2503: the $\BP~ \gexit$ curve.
2504: %
2505: \begin{figure}[hbt]
2506: \centering
2507: \setlength{\unitlength}{0.6bp}
2508: \begin{picture}(200,200)
2509: \put(0,0)
2510: {
2511: \put(10,0){\includegraphics[scale=0.6]{26ebpmapgexit}} %length=180bp
2512: {\tiny
2513: %\footnotesize
2514: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2515: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2516: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2517: }
2518: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2519: \put(22,172){\makebox(0,0)[lb]{$\gexitf{}$}}
2520: \put(90,100){\makebox(0,0)[lb]{{$\gexitf{\BPsmall}(\ih)$}}}
2521: \put(90,80){\makebox(0,0)[lb]{{$=\gexitf{\EBPsmall}(\ih)$}}}
2522: \put(90,60){\makebox(0,0)[lb]{{$=\gexitf{}(\ih)$}}}
2523: \put(46,12){\makebox(0,0)[lb]{$\ih^\BPsmall=\ih^\BPsmall$}}
2524: }
2525: \end{picture}
2526: \caption{EBP GEXIT curve for the cycle-code ensemble with \ddp $(x,x^5)$.
2527: The EBP GEXIT curve, BP GEXIT curve and MAP GEXIT curve coincide. }
2528: \label{fig:26ebpmapgexit}
2529: \end{figure}
2530: \eex
2531:
2532: \bex[(3,6) LDPC Ensemble -- BSC]\label{example:36}
2533: Consider the \ddp $(\ledge,\redge)=(x^2, x^5)$ and the
2534: corresponding $\ldpc$ ensemble with design rate $\drate=1/2$.
2535: We assume that transmission takes place over the family
2536: $\{\text{BSC}(\cp)\}$.
2537: Fig.~\ref{fig:36ebpbpmapgexit} shows on the left the $\EBP$ $\gexit$ curve
2538: and the corresponding $\BP$ $\gexit$ curve, which has one jump.
2539: The picture on the right shows the conjectured
2540: $\MAP$ $\gexit$ curve according to the Maxwell construction.
2541: For this ensemble, we have $\ih^\BPsmall\approx0.416$.
2542: The $\MAP$ threshold implied by the Maxwell construction coincides with
2543: the one of Theorem \ref{theo:UBMAP}:
2544: $\overline{\ih}^\MAPsmall\approx 0.472$.
2545:
2546: \begin{figure}[hbt]
2547: \centering
2548: \setlength{\unitlength}{0.6bp}
2549: \begin{picture}(400,200)
2550: \put(0,0)
2551: {
2552: \put(10,0){\includegraphics[scale=0.6]{36ebpbpgexit}} %length=180bp
2553: {\tiny
2554: %\footnotesize
2555: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2556: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2557: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2558: }
2559: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2560: \put(22,172){\makebox(0,0)[lb]{$\gexitf{}$}}
2561: \put(50,110){\makebox(0,0)[lb]{{$\gexitf{\BPsmall}(\ih)$}}}
2562: \put(66,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2563: }
2564: \put(200,0)
2565: {
2566: \put(10,0){\includegraphics[scale=0.6]{36mapmaxgexit}} %length=180bp
2567: {\tiny
2568: %\footnotesize
2569: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2570: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2571: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2572: }
2573: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2574: \put(22,172){\makebox(0,0)[lb]{$\gexitf {}$}}
2575: \put(105,110){\makebox(0,0)[lb]{{$\gexitf{}(\ih)$}}}
2576: \put(66,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2577: \put(94,12){\makebox(0,0)[lb]{$\ih^\MAPsmall$}}
2578: }
2579: \end{picture}
2580: \caption{EBP GEXIT curve for the $(3,6)$ ensemble. Left: EBP GEXIT curve and corresponding BP GEXIT curve.
2581: Right: The conjectured MAP GEXIT curve according to the Maxwell construction. }
2582: \label{fig:36ebpbpmapgexit}
2583: \end{figure}
2584: \eex
2585: \bex[LDPC$(2/5 x + 3/5 x^5,x^5)$ -- BSC]\label{exampleMult1}
2586: Consider the \ddp $(\ledge,\redge)=(2/5 x + 3/5 x^5,x^5)$ and the
2587: corresponding $\ldpc$ ensemble with design rate $\drate=4/9$.
2588: We assume that transmission takes place over the family
2589: $\{\text{BSC}(\cp)\}$.
2590: Fig.~\ref{fig:stabJmapmaxgexit}
2591: shows on the left the $\EBP$ $\gexit$ curve
2592: and the corresponding $\BP$ $\gexit$ curve, which has one jump.
2593: The picture on the right shows the conjectured
2594: $\MAP$ $\gexit$ curve according to the Maxwell construction.
2595: The $\BP$ threshold is given by the stability condition.
2596: As a consequence of this and Conjecture \ref{con:theconjecture},
2597: $\ih^\BPsmall=\ih^\MAPsmall$ .
2598:
2599: \begin{figure}[hbt]
2600: \centering
2601: \setlength{\unitlength}{0.6bp}
2602: \begin{picture}(400,200)
2603: \put(0,0)
2604: {
2605: \put(10,0){\includegraphics[scale=0.6]{stabJebpbpgexit}} %length=180bp
2606: {\tiny
2607: %\footnotesize
2608: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2609: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2610: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2611: }
2612: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2613: \put(22,172){\makebox(0,0)[lb]{$\gexitf{}$}}
2614: \put(50,110){\makebox(0,0)[lb]{{$\gexitf{\BPsmall}(\ih)$}}}
2615: \put(66,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2616: }
2617: \put(200,0)
2618: {
2619: \put(10,0){\includegraphics[scale=0.6]{stabJmapmaxgexit}} %length=180bp
2620: {\tiny
2621: %\footnotesize
2622: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2623: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2624: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2625: }
2626: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2627: \put(22,172){\makebox(0,0)[lb]{$\gexitf {}$}}
2628: \put(110,110){\makebox(0,0)[lb]{{$\gexitf{}(\ih)$}}}
2629: \put(66,12){\makebox(0,0)[lb]{$\ih^\MAPsmall$}}
2630: }
2631: \end{picture}
2632: \caption{EBP GEXIT curve for the $(\ledge,\redge)=(2/5 x + 3/5 x^5,x^5)$ ensemble. Left: EBP GEXIT curve and corresponding BP GEXIT curve.
2633: Right: The conjectured MAP GEXIT curve according to the Maxwell construction. }
2634: \label{fig:stabJmapmaxgexit}
2635: \end{figure}
2636:
2637: \eex
2638:
2639: \bex[LDPC($\frac{3x+6x^2+11x^{17}}{20},x^9$) -- BSC]
2640: Consider the
2641: \ddp ($\frac{3x+6x^2+11x^{17}}{20},x^9$).
2642: We assume that transmission takes place over the family
2643: $\{\text{BSC}(\cp)\}$.
2644: Fig.~\ref{fig:2Jebpbpgexit} shows on the left the $\EBP$ $\gexit$ curve
2645: and the corresponding $\BP$ $\gexit$ curve, which has two jumps.
2646: The picture on the right shows the conjectured
2647: $\MAP$ $\gexit$ curve according to the Maxwell construction:
2648: This curve has also 2 jumps.
2649: \begin{figure}[hbt]
2650: \centering
2651: \setlength{\unitlength}{0.6bp}
2652: \begin{picture}(400,200)
2653: \put(0,0)
2654: {
2655: \put(10,0){\includegraphics[scale=0.6]{2Jebpbpgexit}} %length=180bp
2656: {\tiny
2657: %\footnotesize
2658: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2659: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2660: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2661: }
2662: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2663: \put(22,172){\makebox(0,0)[lb]{$\gexitf{}$}}
2664: \put(50,110){\makebox(0,0)[lb]{{$\gexitf{\BPsmall}(\ih)$}}}
2665: \put(68,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2666: \put(98,44){\makebox(0,0)[lb]{$\ih^{\BPsmall,2}$}}
2667: }
2668: \put(200,0)
2669: {
2670: \put(10,0){\includegraphics[scale=0.6]{2Jmapmaxgexit}} %length=180bp
2671: {\tiny
2672: %\footnotesize
2673: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2674: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2675: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2676: }
2677: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2678: \put(22,172){\makebox(0,0)[lb]{$\gexitf {}$}}
2679: \put(105,110){\makebox(0,0)[lb]{{$\gexitf{}(\ih)$}}}
2680: \put(66,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2681: \put(96,18){\makebox(0,0)[lb]{$\ih^\MAPsmall$}}
2682: \put(100,58){\makebox(0,0)[lb]{$\ih^{\MAPsmall,2}$}}
2683: }
2684: \end{picture}
2685: \caption{EBP GEXIT curve for the \ddp ($\frac{3x+6x^2+11x^{17}}{20},x^9$). Left: EBP GEXIT curve and corresponding BP GEXIT curve.
2686: Right: The conjectured MAP GEXIT curve according to the Maxwell construction. }
2687: \label{fig:2Jebpbpgexit}
2688: \end{figure}
2689: \eex
2690:
2691: \bex[$(\frac{x+2x^2+2x^{13}}{5},x^5)$ -- BSC] \label{exampleMult3}
2692: Consider the \ddp
2693: $(\frac{x+2x^2+2x^{13}}{5},x^5)$ and the
2694: corresponding $\ldpc$ ensemble. .
2695: We assume that transmission takes place over the family
2696: $\{\text{BSC}(\cp)\}$.
2697: Fig.~\ref{fig:2BP1MAPmapmaxgexit} shows on the left the $\EBP$ $\gexit$ curve
2698: and the corresponding $\BP$ $\gexit$ curve, which has two jumps.
2699: The picture on the right shows the conjectured
2700: $\MAP$ $\gexit$ curve according to the Maxwell construction.
2701: This example shows that a \ddp can have more $\BP$ jumps than
2702: $\MAP$ jumps.
2703:
2704:
2705: \begin{figure}[hbt]
2706: \centering
2707: \setlength{\unitlength}{0.6bp}
2708: \begin{picture}(400,200)
2709: \put(0,0)
2710: {
2711: \put(10,0){\includegraphics[scale=0.6]{2BP1MAPebpbpgexit}} %length=180bp
2712: {\tiny
2713: %\footnotesize
2714: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2715: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2716: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2717: }
2718: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2719: \put(22,172){\makebox(0,0)[lb]{$\gexitf{}$}}
2720: \put(50,110){\makebox(0,0)[lb]{{$\gexitf{\BPsmall}(\ih)$}}}
2721: \put(92,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2722: \put(128,74){\makebox(0,0)[lb]{$\ih^{\BPsmall,2}$}}
2723: }
2724: \put(200,0)
2725: {
2726: \put(10,0){\includegraphics[scale=0.6]{2BP1MAPmapmaxgexit}} %length=180bp
2727: {\tiny
2728: %\footnotesize
2729: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2730: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
2731: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
2732: }
2733: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
2734: \put(22,172){\makebox(0,0)[lb]{$\gexitf {}$}}
2735: \put(128,110){\makebox(0,0)[lb]{{$\gexitf{}(\ih)$}}}
2736: \put(92,12){\makebox(0,0)[lb]{$\ih^\BPsmall$}}
2737: \put(130,18){\makebox(0,0)[lb]{$\ih^\MAPsmall$}}
2738: }
2739: \end{picture}
2740: \caption{EBP GEXIT curve for the \ddp $(\frac{x+2x^2+2x^{13}}{5},x^5)$. Left: EBP GEXIT curve and corresponding BP GEXIT curve.
2741: Right: The conjectured MAP GEXIT curve according to the Maxwell construction. }
2742: \label{fig:2BP1MAPmapmaxgexit}
2743: \end{figure}
2744: \eex
2745:
2746: %*************************************************************************
2747: %
2748: \section{How to Compute
2749: $\EBP$ $\gexit$ Curves: Basic Properties and Area Theorem}
2750: \label{sec:HowToEBP}
2751:
2752: In the previous pages we presented examples
2753: of $\EBP$ $\gexit$ curves for several $\ldpc$ ensembles.
2754: In this section we explain how these curves have been computed
2755: and we derive some of their basic properties,
2756: including the EBP Area Theorem.
2757:
2758: We start by noticing that ordinary density evolution cannot be
2759: applied to the present case because of two reasons. First,
2760: $\EBP$ curves include `unstable branches'. We refer by such a term to
2761: branches along which the $\gexit$ curve is a decreasing
2762: function of the channel entropy. Such branches are expected to
2763: correspond to fixed point densities which are locally unstable under
2764: density evolution (whence the name). This expectation can be confirmed
2765: analytically for the $\BEC$ case, and numerically for a general
2766: $\BMS$ channel. As a consequence, these fixed points cannot be
2767: approximated by iterating density evolution with a generic initial
2768: condition.
2769:
2770: The second problem is related to values of the channel parameter
2771: for which multiple locally stable fixed point densities coexist.
2772: This is the case for instance in the Examples \ref{exampleMult1}
2773: to \ref{exampleMult3} above. In this case different initial conditions
2774: are required to achieve each of these densities by density evolution.
2775: A systematic way for constructing all such initial conditions is however not available.
2776:
2777: The crucial observation for overcoming both these problems consists
2778: in noticing that $\EBP$ $\gexit$ curves are naturally parameterized
2779: by the {\em entropy of the fixed point density}. More precisely,
2780: consider a smooth and degraded family $\{\BMS(\ih)\}$ and
2781: $\xl\in [0,1]$. Then, we expect that there exists at most one
2782: value of the channel parameter $\ih = \ih(\xl)$ and one density
2783: $\Ldens{f}_{\xl}$, such that $H(\Ldens{f}_{\xl})=\xl$ and
2784: $(\Ldens{c}_{\xl} \defas \BMS(\ih(\xl)),\Ldens{f}_{\xl})$ forms a fixed
2785: point pair.
2786:
2787: This naturally suggests to run density evolution {\em at fixed
2788: density entropy}. Let us denote by $\Tc_{\ih}$ the ordinary density evolution
2789: operator at fixed channel $\BMS(\ih)$. Formally
2790: %
2791: \begin{eqnarray}
2792: %
2793: \Tc_{\ih}(\Ldens{a}) \defas \Ldens{c} \conv \ledge(\redge(\Ldens{a}))\, .
2794: %
2795: \end{eqnarray}
2796: %
2797: where $\Ldens{c}$ is the density associated to the channel $\BMS(\ih)$.
2798: For any $\xl\in[0,1]$, we define the density evolution operator at fixed
2799: entropy $\xl$, $\Td_{\xl}$ as
2800: %
2801: \begin{eqnarray}
2802: %
2803: \Td_{\xl}(\Ldens{a}) \defas \Tc_{\ih(\Ldens{a}, \xl)}(\Ldens{a})
2804: %
2805: \end{eqnarray}
2806: %
2807: where $\ih(\Ldens{a}, \xl)$ is the solution of $H(\Tc_{\ih}(\Ldens{a})) =\xl$.
2808: Whenever no such value of $\ih$ exists, $\Td_{\xl}(\Ldens{a})$
2809: is left undefined.
2810: Since, for a given $\Ldens{a}$, the family
2811: $\Tc_{\ih}(\Ldens{a})$ is ordered by physical degradation,
2812: $H(\Tc_{\ih}(\Ldens{a}))$ is a non decreasing function of $\ih$.
2813: As a consequence the equation $H(\Tc_{\ih}(\Ldens{a})) =\xl$
2814: cannot have more than a single solution.
2815: Furthermore, by the smoothness of the channel family
2816: $\BMS(\ih)$, $H(\Tc_{\ih}(\Ldens{a}))$ is continuous.
2817: Notice that
2818: $H(\Tc_{0}(\Ldens{a})) = 0$: if the channel is noiseless the
2819: output density at a variable nodes is noiseless as well.
2820: Therefore, a necessary
2821: and sufficient condition for a solution $\ih(\Ldens{a}, \xl)$
2822: to exist (when the family $\{\BMS(\ent)\}_{\ent}$ is complete) is that
2823: $H(\Tc_{1}(\Ldens{a}))= H(\ledge(\redge(\Ldens{a}))) \ge\xl$.
2824:
2825: Any fixed point of the above transformation $\Td_{\xl}$, i.e. any
2826: $\Ldens{f}$ such that
2827: $\Ldens{f} = \Td_{\xl}(\Ldens{f})$, is also a fixed point of
2828: ordinary density evolution for the channel $\BMS(\ih)$ with
2829: $\ih = \ih(\Ldens{f}, \xl)$, and corresponds to
2830: a point on the $\EBP$ $\gexit$ curve.
2831: Furthermore if a sequence of densities such that $\Ldens{a}_{\ell+1} =
2832: \Td_{\xl}(\Ldens{a}_{\ell})$ converges (weakly)
2833: to a density $\Ldens{f}$, then $\Ldens{f}$ is a fixed point of
2834: $\Td_{\xl}$, with entropy $\xl$.
2835:
2836: This motivates the following numerical procedure which has been used to
2837: determine the $\gexit$ curves plotted in the previous section. $(i)$
2838: Set the initial condition $\Ldens{a}_0 = \BMS(\xl)$.
2839: $(ii)$ For $\ell\ge 0$ compute $\Ldens{a}_{\ell+1} =
2840: \Td_{\xl}(\Ldens{a}_{\ell})$. In practice the convolutions are evaluated
2841: numerically either by sampling or, via Fourier transforms as in ordinary
2842: density evolution. Due to the monotonicity
2843: of $H(T_{\ih}(\Ldens{a}_{\ell}))$ in $\ih$, the value of
2844: $\ih(\Ldens{a}_{\ell}, \xl)$ can be efficiently found by bisection.
2845: $(iii)$ The current estimate of
2846: the $\gexit$ function is given by
2847: $({\ih}_{\ell},\gexitf{\EBPsmall}_{\ell})$.
2848: Here $\ih_\ell\defas\ih(\Ldens{a}_{\ell}, \xl)$
2849: is the current estimate of the channel entropy, and
2850: %
2851: \begin{eqnarray}
2852: %
2853: \gexitf{\EBPsmall}_\ell & \defas
2854: \int_{-\infty}^{\infty} \Ldens{b}_{\ell}(y)\; \gexitkl {\BMS(\ih_{\ell})}
2855: y \;\text{d}y.
2856: %
2857: \end{eqnarray}
2858: %
2859: with $\Ldens{b}_{\ell}\defas \Lambda(\rho(\Ldens{a}_{\ell}))$.
2860: $(iv)$ Halt when some convergence criterion is met and return the
2861: current estimate $({\ih}_{\ell},\gexitf{\EBPsmall}_{\ell})$. In
2862: practice one can require that (a properly defined)
2863: distance between $\Ldens{a}_\ell$ and $\Ldens{a}_{\ell+1}$
2864: becomes smaller than a threshold.
2865:
2866: In all the examples discussed in the previous section, we found
2867: that this procedure converges rapidly, and that the limit point
2868: is (within numerical precision) independent of the initial condition
2869: $\Ldens{a}_0$. Proving these statements seems a challenging task
2870: (notice that unlike in ordinary density evolution, the
2871: sequence $\{\Ldens{a}_{\ell}\}$ is in general not ordered by physical
2872: degradation). However it is easy to show that, if $\xl$ is such that
2873: $\Td_{\xl}$ is `well defined', then this procedure
2874: has at least one fixed point.
2875: %
2876: \btheo\label{theo:ExistenceEntro}
2877: %
2878: Let $(\ledge,\redge)$ be a \ddp, $\xl\in [0,1]$, and $\Td_{\xl}$
2879: the corresponding density evolution operator at fixed density entropy
2880: defined as above,
2881: for the smooth, complete and degraded family $\{\BMS(\ih)\}_{\ih}$.
2882: If $H(\ledge(\redge(\Ldens{a}))) \ge\xl$ for any density $\Ldens{a}$ with
2883: $H(\Ldens{a}) = \xl$, then there exists at least one density
2884: $\Ldens{f}$ such that $\Td_{\xl}(\Ldens{f}) = \Ldens{f}$.
2885: Equivalently, $H(\Ldens{f})=\xl$ and there exists
2886: $\ent\in[0,1]$ such that $\Ldens{f}$ is a fixed point of
2887: density evolution for the channel $\BMS(\ih)$.
2888: %
2889: \etheo
2890: %
2891: \bproof
2892: %
2893: Consider the space ${\sf S}_{\xl}$ of $L$-densities $\Ldens{a}$
2894: such that $H(\Ldens{a}) = \xl$. Any element
2895: in ${\sf S}_{\xl}$ is a probability measure on the completed real line,
2896: satisfying
2897: the symmetry condition (formally $\Ldens{a}(-x) = e^{-x}\Ldens{a}(x)$).
2898: Vice versa, any such probability measure (to be denoted formally
2899: by its `density' $\Ldens{a}$) with $\E [\log(1+e^{-x})] = \xl$) corresponds
2900: to a unique element of ${\sf S}_{\xl}$.
2901: Notice that the completed linear line $\reals_{\infty}$
2902: is a compact metric space (we can for instance identify
2903: it with $[-1,1]$ through the mapping $x\mapsto \tanh(x/2)$ and
2904: use the euclidean metric on $[-1,1]$).
2905: Therefore, the space of probability measure on
2906: $\reals_{\infty}$ is sub-sequentially compact under the weak topology
2907: by Prohorov's theorem \cite{Shir96}.
2908: Both the symmetry condition and $H(\Ldens{a}) = \xl$ are
2909: closed under the same topology, and therefore ${\sf S}_{\xl}$ is compact
2910: as well.
2911:
2912: Let ${\sf BL}$ be the space of bounded Lipshitz function
2913: on $\reals_{\infty}$ (as above, we identify $\reals_{\infty}$
2914: with $[-1,1]$ and consider the Lipschitz condition with respect
2915: to the induced distance) with the corresponding norm $||\cdot ||_{\sf BL}$.
2916: The space of probability measures on $\reals_\infty$
2917: can be viewed as a convex subset of
2918: the dual space ${\sf BL}^*$, and the topology
2919: induced by the dual norm $||\cdot ||_{\sf BL}^*$ coincides with the
2920: weak topology (cf. \cite[Chapter III, \S 7]{Shir96}).
2921: As a consequence ${\sf S}_{\xl}$ is a compact convex subspace of a normed
2922: linear space.
2923:
2924: By hypothesis the mapping $\Ldens{a}\mapsto \Td_{\xl}(\Ldens{a})$,
2925: is well defined for any $\Ldens{a}\in {\sf S}_{\xl}$, and
2926: maps ${\sf S}_{\xl}$ into itself. Furthermore, it is easily seen to
2927: be continuous with respect to the weak topology.
2928: This is a consequence of the Lipschitz continuity of the
2929: functions $(x_1,\dots,x_{\ldegree})\to (x_1+\dots+x_{\ldegree})$
2930: and $(x_1,\dots,x_{\rdegree-1})\to 2\, {\rm atanh}(\tanh(x_1/2)\cdots
2931: \tanh(x_{\rdegree-1}/2))$. Therefore $\Td_{\xl}$ is compact and,
2932: by Schauder's fixed point theorem (cf. \cite{Bro93}, Chapter 4) it has
2933: at least one fixed point.
2934: %
2935: \eproof
2936: %
2937: Notice that the above procedure, as well as Theorem \ref{theo:ExistenceEntro},
2938: holds unchanged if the entropy functional $H(\,\cdot\,)$ is substituted
2939: by any continuous linear functional which preserves physical degradation.
2940:
2941: In checking the hypothesis of Theorem \ref{theo:ExistenceEntro},
2942: as well as in applications, it is important to prove bounds on the
2943: entropy of fixed point pairs $(\Ldens{f},\Ldens{c})$.
2944: We start by recalling upper and lower bounds on the entropy
2945: of $\Tc_{\ent}(\Ldens{a})$ which follows
2946: straightforwardly from \cite{SSZ03,LHHH03,HuH03,LHHH05}.
2947: %
2948: \blemma[Lower Bound]\label{lemmaLBH}
2949: Consider a \ddp $(\ledge, \redge)$ and transmission over the
2950: channel $\BMS(\ent)$. Let
2951: %
2952: \begin{align*}
2953: \underline{l}(\xl) \defas \ledge(\xl), \;\;\;\;\;
2954: \underline{r}(\xl) \defas
2955: \sum_{i} \redge_i
2956: h_2\Bigl(\frac{1-(1-2 \cp(\xl))^{i-1}}{2} \Bigr)\, ,
2957: \end{align*}
2958: %
2959: where $\cp(\xl) \defas h_2^{-1}(\xl)$.
2960: If $\Ldens{a}$ is an $L$-density with $H(\Ldens{a})=\xl$, then
2961: %
2962: \begin{align*}
2963: H(\Tc_{\ent}(\Ldens{a}))\ge \ent\; \underline{l}(\underline{r}(\xl))\, .
2964: \end{align*}
2965: %
2966: \elemma
2967: %
2968: \bproof
2969: %
2970: Following Refs.~\cite{SSZ03,LHHH03,HuH03,LHHH05},
2971: for fixed $H(\Ldens{a})$ and $H(\Ldens{b})$,
2972: $\Ldens{a}\conv \Ldens{b}$ has minimum entropy
2973: if $\Ldens{a}$ and $\Ldens{b}$ are the densities corresponding to a $\BEC$.
2974: On the other hand,
2975: for the convolution at a parity-check node the minimum is achieved
2976: when the input densities correspond to a $\BSC$.
2977: The lemma follows by applying these bounds to random variable
2978: and check nodes with degree distributions given by
2979: $\lambda$ and $\rho$.
2980: %
2981: \eproof
2982: %
2983: This result can be used to check the hypotheses of
2984: Theorem \ref{theo:ExistenceEntro}.
2985: We deduce that, if $\underline{l}(\underline{r}(\xl))\ge \xl$
2986: for some $\xl\in [0,1]$,
2987: then there exists a fixed point pair $(\Ldens{f},\Ldens{c})$
2988: with $H(\Ldens{f}) = \xl$ and $\Ldens{c}=\BMS(\ent)$ for some $\ent$.
2989: For instance, for cycle codes (i.e., for $\lambda(x) =x$) this
2990: implies that such a fixed point pair $(\Ldens{f},\Ldens{c})$
2991: exists for any $H(\Ldens{f})= \xl\in[0,1]$.
2992:
2993: \blemma[Upper Bound]
2994: \label{lemmaUBH}
2995: Consider a \ddp $(\ledge, \redge)$ and transmission
2996: over the channel $\BMS(\ent)$. Let
2997: %
2998: \begin{align*}
2999: %
3000: \overline{l}(\ent, \xl) \defas & \sum_{i} \ledge_i f_{i-1}(\ent, \xl)\, ,
3001: \;\;\;\;\;\;\;\overline{r}(\xl) \defas & 1 - \redge(1-\xl)
3002: %
3003: \end{align*}
3004: %
3005: where
3006: %
3007: \begin{align*}
3008: %
3009: f_i(\ent, \xl) \defas
3010: \sum_{k \in \{ \pm 1\}}
3011: \sum_{j=0}^{i} \binom{i}{j}
3012: (1-\cp(\xl))^{j} \cp(\xl)^{i-j} a_k(\ent)\\
3013: \cdot
3014: \logtwo \Bigl( 1+
3015: \frac{ \cp(\xl)^{2j-i} a_{-k}(\ent)}{(1-\cp(\xl))^{2j-i} a_k(\ent)}
3016: \Bigr)\, ,
3017: %
3018: \end{align*}
3019: %
3020: $a_{+1}(\ent) \defas 1- \cp(\ent)$,
3021: $a_{-1}(\ent) \defas \cp(\ent)$, and $\cp(\ent) \defas h_2^{-1}(\ent)$ as
3022: above.
3023: If $\Ldens{a}$ is an $L$-density with $H(\Ldens{a})=\xl$, then
3024: %
3025: \begin{align*}
3026: H(\Tc_{\ent}(\Ldens{a}))\le \overline{l}(\ent,\overline{r}(\xl))\, .
3027: \end{align*}
3028: %
3029: \elemma
3030: %
3031: \bproof
3032: %
3033: Apply the upper bounds of~\cite{SSZ03,LHHH03,HuH03,LHHH05}
3034: (simply interchange $\BEC$ and $\BSC$).
3035: %
3036: \eproof
3037:
3038: %
3039: \btheo[Bounds on $\exit$ Function]
3040: Consider a \ddp $(\ledge, \redge)$ and transmission
3041: over the degraded family $\{\BMS(\ent)\}_{\ent}$. Define the functions
3042: %
3043: \begin{align*}
3044: %
3045: \underline{L}(\xl) \defas \lnode(\xl)\, , \;\;\;\;\;
3046: \overline{L}(\xl) \defas \sum_{i} \lnode_i f_i(1,\xl),
3047: %
3048: \end{align*}
3049: %
3050: and
3051: $f(\xl,\xl') \defas \max \{\ent: \overline{l}(\ent,\xl')=\xl \}$
3052: (with the convention $f(\xl,\xl')=0$, if the set is empty).
3053: Let $\Ldens{f}$ denote any fixed point of density evolution,
3054: i.e., $\Ldens{f} = \Tc_{\ent}(\Ldens{f})$.
3055: If $\entropy(\Ldens{f})=\xl$ then
3056: %
3057: \begin{align*}
3058: %
3059: f(\xl,\overline{r}(\xl))
3060: \leq \ent \leq
3061: \xl/\underline{l}(\underline{r}(\xl)), \\
3062: \underline{L}(\underline{r}(\xl))
3063: \leq \exitf {\EBPsmall} \leq
3064: \overline{L}(\overline{r}(\xl)).
3065: %
3066: \end{align*}
3067: %
3068:
3069: In words, the entropy parameters of any fixed points of density evolution, and
3070: so in particular the function $\exitf {\EBPsmall}$, are contained in
3071: the union of rectangles as given above.
3072: \etheo
3073: %
3074: \begin{proof}
3075: %
3076: The first two inequality follow from Lemma \ref{lemmaLBH} and \ref{lemmaUBH}.
3077: From Lemma \ref{lemmaLBH} we get
3078: $\xl = H(\Ldens{f}) = H(\Tc_{\ent}(\Ldens{f}))\ge \ent\,
3079: \underline{l}(\underline{r}(\xl))$ which gives the upper bound
3080: on $\ent$. Analogously, Lemma \ref{lemmaUBH} implies
3081: $\xl\ge \overline{l}(\ent,\overline{r}(\xl))$. Since
3082: $\overline{l}(\ent,\overline{r}(\xl))$ is monotonically
3083: increasing in $\ent$, this relation can be inverted as
3084: in the thesis of the theorem.
3085:
3086: Given the fixed point $\Ldens{f}$, the corresponding
3087: $\exit$ entropy at variable nodes is
3088: $\exitf {\EBPsmall} = H(L(\rho(\Ldens{f})))$.
3089: The bounds are obtained as in the proofs of Lemmas \ref{lemmaLBH}
3090: and \ref{lemmaUBH}.
3091: \end{proof}
3092: %
3093: Discussion: The bounds given above are by no means
3094: best possible. First, the given bounds are ``universal'' in
3095: the sense that the are valid for {\em all} channel distributions.
3096: Better bounds for any specific channel family can be derived
3097: by taking the actual input distribution into account.
3098: Even in the universal case slightly better bounds can be given by
3099: taking into account that at the variable node before convolution
3100: with the channel, the incoming message density can not be of arbitrary shape but
3101: that it is already the convolution
3102: of several message densities.
3103: Second, tighter bounds on the extremes of information combining
3104: have been derived in \cite{SSZ05} and can be translated to giver tighter
3105: bounds on $\exit$ functions, albeit at the prize of
3106: more complex expressions. Finally, by using a similar techniques
3107: one can also give bounds on the entropy versus $\gexit$ parameter of
3108: any fixed point with respect to any smooth channel family.
3109:
3110: \bex[LDPC$(2/5 x + 3/5 x^5,x^5)$]
3111: Consider again the \ddp $(\ledge,\redge)=(2/5 x + 3/5 x^5,x^5)$.
3112: Fig.~\ref{fig:exitbound}
3113: shows on the left the construction of the bounded region (union
3114: of rectangles) which contains
3115: all $\EBP$ $\gexit$ curves.
3116: The dashed lines represent the individual curves traced out by the
3117: corner points of the rectangles.
3118: On the right this is compared to the actual
3119: $\EBP$ $\gexit$ curves
3120: for transmission over the $\BSC$ and
3121: the $\BEC$ families (solid lines).
3122: \begin{figure}[hbt]
3123: \centering
3124: \setlength{\unitlength}{0.6bp}
3125: \begin{picture}(400,200)
3126: \put(0,0)
3127: {
3128: \put(10,0){\includegraphics[scale=0.6]{exitbound}} %length=180bp
3129: {\tiny
3130: %\footnotesize
3131: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
3132: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
3133: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
3134: }
3135: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
3136: }
3137: \put(220,0)
3138: {
3139: {\tiny
3140: %\footnotesize
3141: \put(10,0){\includegraphics[scale=0.6]{exitboundpluscurves}} %length=180bp
3142: \multiputlist(20,-2)(32,0)[cb]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
3143: \multiputlist(18,10)(0,32)[rc]{$~$,$0.2$,$0.4$,$0.6$,$0.8$,$1.0$}
3144: \put(18,-2){\makebox(0,0)[rb]{$0.0$}}
3145: }
3146: \put(181,12){\makebox(0,0)[lb]{$\ih$}}
3147: \put(50,110){\makebox(0,0)[lb]{{$\exitf{\BPsmall}(\ih)$}}}
3148: }
3149: \end{picture}
3150: \caption{
3151: Left: Construction of bounding region for all
3152: $\EBP$ $\exit$ curves for the \ddp $(\ledge,\redge)=(2/5 x + 3/5 x^5,x^5)$.
3153: Right: The $\EBP$ $\exit$ curves for transmission over the $\BSC$ and
3154: the $\BEC$ families.
3155: \label{fig:exitbound}}
3156: \end{figure}
3157: \eex
3158: \btheo[EPP Area Theorem]
3159: \label{theo:ebpareatheorem}
3160: Consider the \ddp $(\ledge, \redge)$ and transmission
3161: over the smooth and degraded family $\{\BMS(\ent)\}$.
3162: Let $\gexitf {\EBPsmall}$
3163: denote the corresponding $\EBP$ $\gexit$ function.
3164: Assume that the corresponding $\{\Ldens{f}_\xl\}_\xl$
3165: and $\{\Ldens{c}_\xl\}_\xl$, $\xl \in [0, 1]$,
3166: form a {\em complete fixed-point family}.
3167: Then
3168: \begin{align*}
3169: \int_{0}^1 \gexitf {\EBPsmall}(\xl) \text{d}\xl = 1 -\frac{\int \redge}{\int \ledge}.
3170: \end{align*}
3171: \etheo
3172: \bproof
3173: %\par
3174: \begin{figure}[hbt]
3175: \centering
3176: \setlength{\unitlength}{0.5bp}
3177: \begin{picture}(160,160) % size of small graph
3178: \put(20,0){\includegraphics[scale=0.5]{smallgraph}}
3179: \put(100,10){\makebox(0,0){\small{leaves}}}
3180: \put(100,150){\makebox(0,0){\small{root}}}
3181: \end{picture}
3182: \caption{
3183: Computation tree of depth one for the $(2,4)$-regular LDPC ensemble.}
3184: \vspace{7bp}
3185: \label{fig:smallgraph}
3186: \end{figure}
3187: First, let us assume that the ensemble is $(\ldegree, \rdegree)$-regular. Consider a
3188: variable node and the corresponding computation tree of depth one as shown in
3189: Fig.~\ref{fig:smallgraph}.
3190: Let us assume that the bit associated to the
3191: root node is passed through the channel characterized by $\Ldens{c}_\xl$, while the ones
3192: associated to the leaf nodes are passed through a channel
3193: characterized by $\Ldens{f}_\xl$.
3194: Apply the GAT:
3195: let $X=(X_1,\dots,X_{1+\ldegree\times(\rdegree-1)})$
3196: be the transmitted codeword chosen uniformly at random
3197: from the tree code and $Y(\xl)$ be the result of
3198: passing the bits of $X$ through their respective channels
3199: with parameter $\xl$.
3200: Note that $H(X \mid Y(\xl=1))-H(X \mid Y(\xl=0)) =H(X)$. This
3201: follows since by assumption the fixed-point family is complete.
3202: In particular this implies that the channel for $\xl=0$ is
3203: the ``noiseless'' channel so that $H(X \mid Y(\xl=0))=0$.
3204: By the GAT, this difference is equal to the sum of the integrals
3205: of the individual $\gexitfi {}$ curves, where the integral extends from $\xl=0$ to $\xl=1$.
3206: There are two types of individual $\gexitfi {}$ curves, namely the one
3207: associated to the root node, call it $\gexitfroot {}$,
3208: and the $\ldegree (\rdegree-1)$ ones associated to the leaf nodes,
3209: call them $\gexitfleaf {}$. To summarize, the GAT
3210: states
3211: \begin{align*}
3212: H(X) & = \int_0^1 \gexitfroot {} (\xl)\, \text{d}\xl
3213: +\ldegree (\rdegree-1) \int_0^1\gexitfleaf {} (\xl)\text{d}\xl.
3214: \end{align*}
3215: Note that $H(X) = 1+\ldegree (\rdegree-1)-\ldegree=1-\ldegree (\rdegree-2)$ since the computati
3216: on tree
3217: contains $1+\ldegree(\rdegree-1)$ variable nodes and $\ldegree$ check nodes.
3218: Moreover, $\int_0^1 \gexitfleaf {} (\xl)\text{d}\xl=\int_0^1 1-\redge(1-\xl)\text{d}\xl=
3219: ({\rdegree-1})/{\rdegree}$. This follows by applying the GAT once
3220: again to a $[\rdegree, 1, \rdegree-1]$ single parity check code.
3221: Collecting these observations and solving for
3222: $\int_{0}^{1} \gexitfroot {} (\xl) \;\text{d}\xl$, we get
3223: \begin{align*}
3224: \int_{0}^{1} \gexitfroot {}(\xl) \;\text{d}\xl = 1- \ldegree/\rdegree=\drate,
3225: \end{align*}
3226: as claimed since $\gexitfroot{} =\gexitf {\EBPsmall}$.
3227:
3228: The irregular case follows in the same manner: we consider the ensemble of computation
3229: trees of depth one where the degree of the root note is chosen according to the
3230: node degree distribution $\lnode$ and each edge emanating from this root node
3231: is connected to a check node whose degree is chosen according to the edge degree distribution
3232: $\redge$. As before, leaf nodes experience the channel characterized by $\Ldens{f}_x$,
3233: whereas the root node experiences the channel characterized
3234: by $\Ldens{c}_x$. We apply the GAT to each such choice and
3235: average with the respective probabilities.
3236: \end{proof}
3237:
3238: This result imposes some strong constraint on $\BP$ $\gexit$ functions
3239: and their relation to $\MAP$ $\gexit$ functions. Here is an example.
3240: %
3241: \begin{corollary}\label{CoroCycle}
3242: Consider communication over the smooth and degraded family
3243: $\{\BMS(\ent)\}_{\ent}$, $\ent\in[0,1]$ using uniformly
3244: random codes from the ensemble $\eldpc n \ledge \redge$ and assume that the
3245: rate of this ensemble converges to the design rate, see \cite[Lemma 7]{MMU05}.
3246: Assume that the $\BP$ fixed point family
3247: $\{\BMS(\ent),\Ldens{a}_{\ent}\}$, is smooth and complete.
3248: Then ($\MAP$) $\gexit$ function and $\BP$ $\gexit$ function
3249: coincide: $\gexitf{}(\ent)=\gexitf{\BPsmall}(\ent)$ for almost every
3250: $\ent\in[0,1]$.
3251: \end{corollary}
3252: %
3253: \begin{proof}
3254: %
3255: By hypothesis we can apply Theorem \ref{theo:ebpareatheorem} to
3256: the $\BP$ $\gexit$ function. We get
3257: %
3258: \begin{eqnarray*}
3259: %
3260: \int_0^1\gexitf{\BPsmall}(\ent)\, \de\ent = r\, .
3261: %
3262: \end{eqnarray*}
3263: %
3264: Further, by the GAT (and applying Fubini theorem and
3265: Fatou's lemma as in the proof of Theorem \ref{theo:UBMAP})
3266: %
3267: \begin{eqnarray*}
3268: %
3269: \int_0^1\gexitf{}(\ent)\, \de\ent = r\, .
3270: %
3271: \end{eqnarray*}
3272: %
3273: The proof if completed by noticing that, because of Lemma
3274: \ref{lemma:gexitmapversusgexitbp}, $\gexitf {}(\ent)
3275: \leq \gexitf {\BPsmall}(\ent)$ for every $\ent\in[0,1]$.
3276: %
3277: \end{proof}
3278: %
3279: Proving that the hypotheses of this Corollary hold for some
3280: \ddp $(\ledge,\redge)$ is a challenging task
3281: (see also next section). On the other hand, numerical computations
3282: show very clearly that this is the case, for instance, for cycle ensembles,
3283: cf. Example \ref{ex:CycleEx}.
3284: %
3285: %***********************************************************************
3286: %
3287: \section{Regularity of Extended BP $\gexit$ Curves}
3288: \label{sec:Regularity}
3289:
3290: Theorem \ref{theo:ExistenceEntro} ensures (for many $\ldpc$
3291: ensembles) the existence of a fixed point pair
3292: $(\Ldens{f}_{\xl},\Ldens{c}_{\xl})$ for each value of
3293: $\xl = H(\Ldens{f}_{\xl})$. However, for applying the extended Area
3294: Theorem \ref{theo:ebpareatheorem} the resulting family has to be smooth with
3295: respect to the parameter $\xl$. That this is indeed the case
3296: is strongly suggested by the numerical computation of the $\EBP$ curve, cf. Sec.~\ref{sec:egexit}.
3297: We provide here some partial analytic results in this direction.
3298:
3299: Throughout this section, we denote by $\batta(\Ldens{a})$ the Battacharyya
3300: parameter for the $L$-density $\Ldens{a}$. Furthermore, when assuming
3301: communication through the channel $\BMS(\ih)$, we denote by
3302: $B_{\ih}$ the Battacharyya parameter of the channel.
3303: %
3304: \begin{lemma}
3305: %
3306: Assume communication over the degraded family $\{\BMS(\ih)\}_{\ih}$ channel using the
3307: \ddp $(\lambda,\rho)$. Then, for any $\ih$, there exists at most a unique fixed point
3308: density $\Ldens{f}_{\ih}$ such that
3309: %
3310: \begin{eqnarray}
3311: %
3312: B_{\ih}\lambda'(1)\rho''(1-\batta(\Ldens{f}_{\ih})^2)<1
3313: \, .\label{eq:UniquenessCond}
3314: %
3315: \end{eqnarray}
3316: %
3317: Furthermore, if such a density $\Ldens{f}_{\ih}$ exists, it coincides with
3318: the BP fixed point. Finally,
3319: $\batta(\Ldens{f}_{\ih})$ is Lipschitz continuous with respect to $B_\ih$.
3320: More precisely, if the two fixed points
3321: $\Ldens{f}_{\ih_1}$, $\Ldens{f}_{\ih_2}$ satisfy the condition
3322: $B_{\ih_i}\lambda'(1)\rho''(1-\batta(\Ldens{f}_{\ih_i})^2)\le 1-\delta$
3323: for some $\delta>0$, then there exists $C = C(\delta,\lambda,\rho)$,
3324: such that
3325: %
3326: \begin{eqnarray*}
3327: %
3328: |\batta(\Ldens{f}_{\ih_1})-\batta(\Ldens{f}_{\ih_2})|\le C\,|B_{\ih_1}-B_{\ih_2}|\, .\label{eq:Lipschitz}
3329: %
3330: \end{eqnarray*}
3331: %
3332: \end{lemma}
3333: %
3334: \begin{proof}
3335: %
3336: Consider two channel parameters $\ih_1\le\ih_2$ and
3337: two $L$-densities $\Ldens{a}_1$ and
3338: $\Ldens{a}_2$ satisfying the condition
3339: $B_{\ih_i}\lambda'(1)\rho''(1-\batta(\Ldens{f}_{\ih_i})^2)<1-\delta$
3340: for some $\delta>0$.
3341: Assume that $\Ldens{a}_2$ is physically degraded with
3342: respect to $\Ldens{a}_1$. We prove in Appendix \ref{sec:ProofContraction}
3343: that there exists a constant $\alpha=\alpha(\lambda,\rho,\delta)<1$
3344: on $\delta$, the channel family and the degree distribution, such that
3345: %
3346: \begin{align}
3347: %
3348: |\batta(\Tc_{\ih_1}(\Ldens{a}_{1}))-&\batta(\Tc_{\ih_2}(\Ldens{a}_{2}))|
3349: \le \label{eq:Contraction}\\ &\alpha\,
3350: |\batta(\Ldens{a}_{1})-\batta(\Ldens{a}_{2})|
3351: +|B_{\ih_1}-B_{\ih_2}|\, .\nonumber
3352: %
3353: \end{align}
3354: %
3355:
3356: Let us show that this result implies the thesis.
3357: Denote by $\Ldens{f}_{\ih}$ the BP fixed point for the channel
3358: $\BMS(\ih)$ and notice that any other fixed point $\Ldens{f}'_{\ih}$
3359: for the same channel is necessarily physically upgraded with respect to
3360: $\Ldens{f}_{\ih}$. Using the standard notation
3361: $\Ldens{f}_{\ih}\succ \Ldens{f}'_{\ih}$.
3362: In fact $\Delta_0\succ\Ldens{f}'_{\ih}$. By applying the
3363: density evolution operator, we deduce that
3364: $\Ldens{a}^{\BPsmall,\ell}_{\ih}\succ\Ldens{f}'_{\ih}$,
3365: where $\Ldens{a}^{\BPsmall,\ell}$ is the density after $\ell$ iterations of BP.
3366: By taking the limit $\ell\to\infty$ we get $\Ldens{f}_{\ih}\succ
3367: \Ldens{f}'_{\ih}$.
3368:
3369: Next notice that, if $\Ldens{f}_\ih$ satisfies Eq.~(\ref{eq:UniquenessCond})
3370: there cannot be a distinct fixed point, physically upgraded with respect
3371: to $\Ldens{f}_{\ih}$, also satisfying
3372: Eq.~(\ref{eq:UniquenessCond}). If such a density $\Ldens{f}_{\ih}'$ existed,
3373: we could apply (\ref{eq:Contraction}) to get
3374: %
3375: \begin{align*}
3376: %
3377: |\batta(T_{\ih}(\Ldens{f}_{\ih}))-\batta(T_{\ih}(\Ldens{f}'_{\ih}))|
3378: \le \alpha\,
3379: |\batta(\Ldens{f}_{\ih})-\batta(\Ldens{f}'_{\ih})|\, ,
3380: %
3381: \end{align*}
3382: %
3383: with $\alpha<1$. But, since
3384: $T_{\ih}(\Ldens{f}_{\ih}) = \Ldens{f}_{\ih}$ and
3385: $T_{\ih}(\Ldens{f}'_{\ih}) = \Ldens{f}'_{\ih}$, this would imply
3386: $\batta(\Ldens{f}_{\ih})=\batta(\Ldens{f}'_{\ih})$ which is
3387: impossible because $\Ldens{f}_{\ih}\succ
3388: \Ldens{f}'_{\ih}$.
3389:
3390: Let us finally prove Lipschitz continuity, cf. Eq.~(\ref{eq:Lipschitz}).
3391: Under our hypotheses,
3392: the two fixed points $\Ldens{f}_{\ih}$, $\Ldens{f}_{\ih'}$ are
3393: the BP fixed points for channels $\BMS(\ent)$ and $\BMS(\ent')$.
3394: Consider therefore the BP sequences
3395: $\{\Ldens{a}^{\BPsmall,\ell}_{\ih}\}_{\ell\ge 0}$,
3396: $\{\Ldens{a}^{\BPsmall,\ell}_{\ih'}\}_{\ell\ge 0}$. For each $\ell$,
3397: $\Ldens{a}^{\BPsmall,\ell}_{\ih}$ (respectively $\Ldens{a}^{\BPsmall,\ell}_{\ih'}$)
3398: is physically degraded with respect to $\Ldens{f}_{\ih}$ (respectively
3399: $\Ldens{f}_{\ih'}$),
3400: and therefore satisfies the condition (\ref{eq:UniquenessCond}),
3401: since the latter does.
3402: Furthermore, assuming without loss of generality
3403: $\ent'>\ent$, we have $\Ldens{a}^{\BPsmall,\ell}_{\ih'}\succ
3404: \Ldens{a}^{\BPsmall,\ell}_{\ih}$.
3405: Let $\delta_{\ell} \defas
3406: |\batta(\Ldens{a}^{(\ell)}_{\ih}-\batta(\Ldens{a}^{(\ell)}_{\ih'})|$.
3407: Clearly $\delta_0=0$. By applying Eq.~(\ref{eq:Contraction}), we
3408: get
3409: $\delta_{\ell+1}\le \alpha\, \delta_{\ell}
3410: +|B_{\ih_1}-B_{\ih_2}|$, and therefore
3411: %
3412: \begin{align*}
3413: %
3414: \delta_{\ell}\le (\alpha+\alpha^2+\cdots+\alpha^{\ell})\,
3415: |B_{\ih_1}-B_{\ih_2}|\le \frac{\alpha}{1-\alpha}\, |B_{\ih_1}-B_{\ih_2}|\, .
3416: %
3417: \end{align*}
3418: %
3419: The thesis follows by taking the $\ell\to\infty$ limit.
3420: %
3421: \end{proof}
3422: %
3423: It is worth mentioning that the Lipschitz condition Eq.~(\ref{eq:Lipschitz})
3424: implies analogous regularity properties for other functionals
3425: of the density $\Ldens{a}_{\ih}$. For instance, it is easy to show that
3426: $|H(\Ldens{f}_{\ih_1})-H(\Ldens{f}_{\ih_2})|\le A \,
3427: |\batta(\Ldens{f}_{\ih_1})-\batta(\Ldens{f}_{\ih_2})|$, for some
3428: universal constant $A$. Also, the Battacharyya parameter is, for most
3429: channel families, a smooth function of the channel parameter. Regularity
3430: with respect to $B_{\ih}$ translates therefore immediately into regularity
3431: with respect to $\ih$.
3432:
3433: In applying the above result, it is helpful to have bounds on the
3434: Battacharyya parameter of the fixed point densities.
3435: %
3436: \begin{lemma}\label{lemma:LB_Bhatta}
3437: %
3438: Assume communication over the channel $\BMS(\ih)$ using random
3439: codes from the $(\lambda,\rho)$ ensemble. If $\Ldens{f}$ is a fixed point
3440: density with Battacharyya parameter $b=\batta(\Ldens{f})$, then
3441: %
3442: \begin{align*}
3443: %
3444: b\ge B_{\ih}\lambda(\tilde{b})\, ,\;\;\;\;\;
3445: \tilde{b} \defas \sum_{\rdegree}\rho_{\rdegree}\sqrt{1-(1-b^2)^{\rdegree-1}}\,.
3446: %
3447: \end{align*}
3448: %
3449: %
3450: \end{lemma}
3451: %
3452: \begin{proof}
3453: %
3454: First notice that $b = B_{\ih}\lambda(\tilde{b}')$ where
3455: $\tilde{b} \defas \sum_{\rdegree}\rho_{\rdegree}
3456: \batta(\Ldens{f}^{\boxast (\rdegree-1)})$, and $\boxast$ denotes the
3457: convolution at check nodes. It
3458: is convenient to write densities in terms of the variable
3459: $u\defas\sqrt{1-\tanh^2(x/2)}$. With a slight abuse of notation,
3460: we use the same symbol $\Ldens{f}$ to denote the density with respect to
3461: $u$. We get
3462: %
3463: \begin{align*}
3464: %
3465: \batta(\Ldens{f}^{\boxast i})\!=\!\!
3466: \int \!\sqrt{1-(1-u^2_1)\cdots(1-u_i)^2}\,
3467: \Ldens{f}(u_1)\de u_1\cdots\Ldens{f}(u_i)\de u_i\, .
3468: %
3469: \end{align*}
3470: %
3471: The proof is completed by using convexity with respect to the
3472: $u_1$,\dots $u_i$ together with the fact that
3473: $b=\int \!u \Ldens{f}(u)\,\de u$.
3474: %
3475: \end{proof}
3476:
3477: \bex
3478: Consider the $(2,3)$ ensemble and communication over the $\BSC(\epsilon)$.
3479: In this case Eq.~(\ref{eq:UniquenessCond}) is equivalent to
3480: %
3481: \begin{eqnarray}
3482: %
3483: \batta(\Ldens{f})>\sqrt{1-\frac{1}{2B(\epsilon)}}\, ,\label{eq:Uniq23}
3484: %
3485: \end{eqnarray}
3486: %
3487: where $B(\epsilon)$ denotes the channel Battacharyya parameter as a
3488: function of the flip probability $B(\epsilon) = \sqrt{4\epsilon(1-\epsilon)}$.
3489: Lemma \ref{lemma:LB_Bhatta} implies (if we neglect the
3490: case $\batta(\Ldens{f})=0$ which corresponds to a no-error
3491: fixed point) $\batta(\Ldens{f})\ge \sqrt{2-B(\epsilon)^{-2}}$.
3492: This lower bound lies in the region described by equation (\ref{eq:Uniq23})
3493: as soon as $B(\epsilon)\ge (\sqrt{17}-1)/4$, i.e., $\epsilon>\epsilon_*$
3494: with
3495: %
3496: \begin{eqnarray*}
3497: %
3498: \epsilon_* = \frac{1}{2}-\sqrt{\frac{\sqrt{17}-1}{32}}\, ,
3499: %
3500: \end{eqnarray*}
3501: %
3502: which yields $\epsilon_*\approx 0.18759473$. The above results imply that
3503: a unique fixed point density (apart from the no-error one) exists
3504: for any $\epsilon>\epsilon_*$. On the other hand numerical computations
3505: suggest this to be the case for all values above the local stability
3506: threshold $\epsilon_{\rm ls} = (2-\sqrt{2})/4\approx 0.066987298$.
3507: Fig.~\ref{fig:23example} shows the Battacharyya constant of the fixed point
3508: density as a function of the channel parameter of the \BSC\ (solid line),
3509: the bound stated in (\ref{eq:Uniq23}) (dotted line), as well as
3510: the bound $\batta(\Ldens{f})\ge \sqrt{2-B(\epsilon)^{-2}}$ (dashed line).
3511: \begin{figure}[hbt]
3512: \centering
3513: \setlength{\unitlength}{0.5bp}
3514: \begin{picture}(170,170)
3515: \put(0,0){\includegraphics[scale=0.5]{23example}}
3516: {\footnotesize
3517: \multiputlist(0,-4)(32,0)[tc]{$0.0$,$0.1$,$0.2$,$0.3$,$0.4$, $0.5$}
3518: \put(170,0){\makebox(0,0)[lb]{$\epsilon$}}
3519: }
3520: \end{picture}
3521: \caption{\label{fig:23example}
3522: The solid line shows the Battacharyya constant of the fixed point
3523: density as a function of the channel parameter of the \BSC.
3524: The dotted line corresponds to the bound stated in (\ref{eq:Uniq23}),
3525: whereas the dashed curve corresponds to the bound $\batta(\Ldens{f})\ge \sqrt{2-B(\epsilon)^{-2}}$.}
3526: \end{figure}
3527: %
3528: \eex
3529: %
3530:
3531: As stressed in the previous section, $\EBP$ curves are expected to be
3532: single valued smooth functions of the entropy $H(\Ldens{f})$ of the
3533: fixed point density. The same expectation holds, if entropy is replaced by
3534: any linear functional which preserves physical degradation.
3535: The following result confirms that better regularity properties can
3536: indeed be obtained by taking this point of view.
3537: %
3538: \begin{lemma}
3539: Let $\{\BMS(\ent)\}_{\ent}$ be a degraded family.
3540: Assume $\Ldens{f}_1$ and $\Ldens{f}_2$ to be fixed point densities
3541: for the channel parameters $\ih_1$, $\ih_2$, and that
3542: $\Ldens{f}_1$ is physically degraded with respect to $\Ldens{f}_2$.
3543: If $\batta(\Ldens{f}_1)\ge\delta>0$, then there exists a constant
3544: $C=C(\lambda,\rho,\delta)$ such that
3545: %
3546: \begin{eqnarray*}
3547: %
3548: |B_{\ih_1}-B_{\ih_2}|\le C\, |\batta(\Ldens{f}_1)-\batta(\Ldens{f}_2)|\, .
3549: %
3550: \end{eqnarray*}
3551: %
3552: In words, the channel is a Lipschitz continuous function
3553: of the Battacharyya parameter of the fixed point density.
3554: %
3555: \end{lemma}
3556: %
3557: \begin{proof}
3558: %
3559: Proceeding as in the proof of Eq.~(\ref{eq:Contraction}),
3560: cf.~Appendix \ref{sec:ProofContraction} it is easy to show
3561: that, if $\Ldens{a}_1\succ\Ldens{a}_2$, then
3562: %
3563: \begin{align*}
3564: %
3565: |\batta(\Tc_{1}(\Ldens{a}_{1}))-\batta(\Tc_1(\Ldens{a}_{2}))|
3566: \le \lambda'(1)\rho''(1)|\batta(\Ldens{a}_{1})-\batta(\Ldens{a}_{2})|\, .
3567: %
3568: \end{align*}
3569: %
3570: Furthermore, if $\batta(\Ldens{a})\ge\delta>0$, then
3571: $\batta(\Tc_1(\Ldens{a}))\ge\delta'$ for some $\delta'>0$.
3572:
3573: Consider now the difference $|\batta(\Tc_{\ih_1}(\Ldens{f}_1))-
3574: \batta(\Tc_{\ih_2}(\Ldens{f}_2))|$. Since
3575: $\Ldens{f}_{1/2}$ are density evolution fixed points, this is equal
3576: to $|\batta(\Ldens{f}_1)-\batta(\Ldens{f}_2)|$.
3577: We get therefore
3578: %
3579: \begin{align*}
3580: %
3581: |\batta&(\Ldens{f}_1)-\batta(\Ldens{f}_2)| =
3582: |B_{\ih_1}\batta(\Tc_{1}(\Ldens{f}_1))-B_{\ih_2}\batta(\Tc_{1}(\Ldens{f}_2))|\\
3583: \ge & |B_{\ih_1}-B_{\ih_2}|\batta(\Tc_{1}(\Ldens{f}_2))-
3584: B_{\ih_1}|\batta(\Tc_{1}(\Ldens{f}_1))-\batta(\Tc_{1}(\Ldens{f}_2))|\\
3585: \ge & |B_{\ih_1}-B_{\ih_2}|\delta'-
3586: \lambda'(1)\rho''(1)|\batta(\Ldens{f}_{1})-\batta(\Ldens{f}_{2})|\,,
3587: %
3588: \end{align*}
3589: %
3590: which implies the thesis after solving for $|B_{\ih_1}-B_{\ih_2}|$.
3591: %
3592: \end{proof}
3593: %
3594:
3595: %
3596: %***********************************************************************
3597: %
3598: \section{$\MAP$ Versus $\BP$ Marginals}
3599: \label{sec:mapversusbp}
3600:
3601: As we saw in Sections \ref{sec:egexit} and \ref{sec:HowToEBP}, the
3602: $\MAP$ and $\BP$ $\gexit$ curves are strictly related for
3603: LDPC codes in the large blocklength limit.
3604: We conjectured that they can be connected through the
3605: Maxwell construction.
3606: In particular, this would imply that they are asymptotically equal above the
3607: $\MAP$ threshold for a large family of ensembles, cf. for instance
3608: Example \ref{example:36}.
3609:
3610: Does the coincidence of $\gexit$ curves mean that
3611: $\BP$ and $\MAP$ decoding in fact coincide {\em bit by bit}?
3612: More precisely, belief propagation can be
3613: regarded as a low complexity (approximate) algorithm for computing
3614: the marginal distributions $p_{X_i\mid Y}(x_i\mid y)$. It is well established
3615: \cite{RiU01}, that the $\BP$ estimate is asymptotically correct in the
3616: low noise regime $\ent<\ent_{\BP}$. We wonder whether the same is true
3617: whenever the two $\gexit$ functions coincide.
3618:
3619: Perhaps surprising, the answer is positive. In order to proceed, it is
3620: convenient to introduce some notations. For the sake of simplicity
3621: we consider the case of a binary channel. Rather than the
3622: marginal distributions $p_{X_i|Y}(x_i\mid y)$, it is convenient to
3623: focus on the extrinsic soft bits
3624: %
3625: \begin{align*}
3626: %
3627: \mu_{i}(y) \equiv \E[X_i\mid Y_{\sim i} = y_{\sim i}]\, .
3628: %
3629: \end{align*}
3630: %
3631: We will further denote by $\mu_{i}^{\BPsmall,\ell}(y)$,
3632: the estimate of this quantity
3633: provided by $\BP$, after $\ell$ iterations.
3634: Notice that $\mu_{i}(y) = \tanh\phi_i^{}(y_{\sim i})$,
3635: and $\mu_{i}^{\BPsmall,\ell}(y) = \tanh\phi_i^{\BPsmall,\ell}(y_{\sim i})$.
3636:
3637: A meaningful measure of how much `incorrect' is $\BP$, is the mean square
3638: error
3639: %
3640: \begin{align*}
3641: %
3642: \Delta^{(\ell)}(y) \equiv\frac{1}{n}\sum_{i=1}^n
3643: \left|\mu_i^{\BPsmall,\ell}(y)-\mu_i(y)\right|^2\, .
3644: %
3645: \end{align*}
3646: %
3647: Let us stress that $\Delta^{(\ell)}(y)$ implies a rather strict notion
3648: of correctness. We are not just requiring the hard decision
3649: reached by $\BP$ to be (approximatively) the same that would be provided
3650: by a $\MAP$ decoder. Rather, $\BP$ should be able to reconstruct the
3651: full information about $X_i$, given the received message.
3652:
3653: Our main result is presented below (here we refer to the Tanner graph
3654: associated to the code parity check matrix, which is naturally related to
3655: belief propagation).
3656: %
3657: \begin{theorem}\label{thm:BPcorrect}
3658: %
3659: Consider communication using a linear code over a smooth channel $\BMS(\ent)$,
3660: and let $Y$ be the channel output if the input is uniformly random codeword
3661: $X$.
3662: Let $\gexitkabsd{}{\cdot}$ denote the $\gexit$ kernel in the $|D|$-domain
3663: and $K \equiv -\sup\left\{\frac{\text{d}^2\gexitkabsd{}{x}}{\text{d} x^2} :\, x\in[0,1]\right\}>0$.
3664: Assume that, for a uniformly
3665: random variable node $i$ in the Tanner graph, the shortest loop
3666: through $i$ has length larger than $2\ell$ with probability at least
3667: $1-\delta$. Then
3668: %
3669: \begin{eqnarray*}
3670: %
3671: \E\, \Delta^{(\ell)}(Y)\le \frac{2}{K}[\gexitf{\BPsmall,\ell}(\ent)-
3672: \gexitf{}(\ent)] +4\delta\, .
3673: %
3674: \end{eqnarray*}
3675: %
3676: \end{theorem}
3677: %
3678: Let us stress that this result holds, not just for random elements
3679: of an $\eldpc n \ledge \redge$ ensemble, but for any code with the prescribed
3680: sparseness properties.
3681:
3682: The proof makes use of a technical lemma, which we state below, and prove
3683: in Appendix~\ref{app:BPcorrect}.
3684: %
3685: \blemma\label{lem:GEXITSquareError}
3686: Consider a random variable $X$ taking values in $\{+1,-1\}$ and
3687: assume that $X\to Y\to Z$ forms a Markov chain. Let
3688: $k:[0,1]\to {\mathbb R}$ be twice differentiable with
3689: $k'(0)\le 0$, and $k''(x)\le -K<0$ for any $x\in [0,1]$.
3690: If we denote $\mu_{{\sf Y}}(y) = \E[X\mid Y=y]$ and
3691: $\mu_{{\sf Z}}(z) = \E[X\mid Z=z]$,
3692: then
3693: %
3694: \begin{align*}
3695: %
3696: \E[k(|\mu_{{\sf Y}}(Y)|)] \le\E[k(|\mu_{{\sf Z}}(Z)|)]
3697: -\frac{1}{2}\,K\,\E[|\mu_{{\sf Y}}(Y)-\mu_{{\sf Z}}(Z)|^2]\, .
3698: %
3699: \end{align*}
3700: %
3701: \elemma
3702:
3703: \begin{proof}[Theorem \ref{thm:BPcorrect}]
3704: The $\MAP$ $\gexit$ function can be written as
3705: %
3706: \begin{eqnarray*}
3707: %
3708: \gexitf{}(\ent) = \frac{1}{n}\sum_{i=1}^n\,
3709: \E[\gexitkabsd{}{|\mu_i(Y)|}]\, .
3710: %
3711: \end{eqnarray*}
3712: %
3713: An analogous expression holds for the $\BP$ $\gexit$ function if we replace
3714: $\mu_i(Y)$ with $\mu^{\BPsmall,\ell}_i(Y)$. We claim that,
3715: if the shortest loop through $i$ in the Tanner graph is longer than
3716: $2\ell$, then
3717: %
3718: \begin{align}
3719: %
3720: \E[\gexitkabsd{}{|\mu_i(Y)|}]\le&\E
3721: [\gexitkabsd{}{|\mu^{\BPsmall,\ell}_i(Y)|}]-\label{eq:LocalErr}\\
3722: &-\frac{1}{2}\,K\, \E[(\mu_i(Y)-\mu^{\BPsmall,\ell}_i(Y))^2]\, .\nonumber
3723: %
3724: \end{align}
3725: %
3726: The thesis follows by rearranging the terms, using the trivial
3727: bound $(\mu_i(Y)-\mu^{\BPsmall,\ell}_i(Y))^2\le 4$ whenever the
3728: shortest loop through $i$ is not longer than $2\ell$ and summing over $i$.
3729:
3730: In order to prove the above claim, let $Y_{\sim i}^{(\ell)}$ denote the subset
3731: of received signals within a distance $\ell$ from the variable node $i$
3732: on the Tanner graph. Notice that $X_i\to Y_{\sim i}\to Y_{\sim i}^{(\ell)}$
3733: is a Markov chain, and that $\mu_i(Y) = \E[X_i|Y_{\sim i}]$,
3734: $\mu^{(\ell)}_i(Y) = \E[X_i|Y_{\sim i}^{(\ell)}]$.
3735: We can therefore apply Lemma~\ref{lem:GEXITSquareError}, with
3736: $k(x) =\gexitkabsd{}{x}$. This yields Eq.~(\ref{eq:LocalErr}), and thus
3737: concludes the proof.
3738: \end{proof}
3739: One may wonder whether the distortion measure $\Delta^{(\ell)}(y)$
3740: is appropriate. One could, for instance consider the
3741: actual soft bits, rather than the extrinsic ones. If we let
3742: $\tilde{\mu}_i(y) = \E[X_i|Y=y]$, and denote as
3743: $\tilde{\mu}^{\BPsmall,\ell}_i(y)$ the corresponding $\BP$
3744: estimate, we may define
3745: %
3746: \begin{eqnarray*}
3747: %
3748: \widetilde{\Delta}^{(\ell)}(y) = \frac{1}{n}
3749: \sum_{i=1}^n\left|\tilde{\mu}^{\BPsmall,\ell}_i(y)-\tilde{\mu}_i(y)
3750: \right|^2\, .
3751: %
3752: \end{eqnarray*}
3753: %
3754: Recall that hard decoding decisions are taken in terms of $\tilde{\mu}_i(y)$,
3755: rather than $\mu_i(y)$. We are therefore interested in knowing whether
3756: $\widetilde{\Delta}^{(\ell)}(y)$ can be much larger than
3757: $\Delta^{(\ell)}(y)$. The answer is generically negative, as shown by the
3758: lemma below.
3759: %
3760: \begin{lemma}\label{lem:ExtrinsicDistortion}
3761: Assume communication over a $\BMS$ channel with $L$-density
3762: $\Ldens{c}(l)$. Then
3763: %
3764: \begin{eqnarray*}
3765: %
3766: \E\,\widetilde{\Delta}^{(\ell)}(Y)\le C\, \E\,\Delta^{(\ell)}(Y)
3767: %
3768: \end{eqnarray*}
3769: %
3770: where $C \equiv \int\! e^{2|l|}\Ldens{c}(l) \, \text{d}l$.
3771: \end{lemma}
3772: %
3773: The proof is deferred to Appendix~\ref{app:BPcorrect}.
3774:
3775: Theorem \ref{thm:BPcorrect} obviously imply that belief propagation
3776: is `asymptotically correct' every time the $\BP$ and $\MAP$ $\gexit$ functions
3777: asymptotically coincide. We conjectured in Section \ref{sec:egexit} that
3778: the $\MAP$ $\gexit$ function can be obtained from the $\EBP$ one through
3779: the Maxwell construction. This construction allows therefore to determine in
3780: which domain of $\ent$ $\BP$ and $\MAP$ $\gexit$ functions do coincide.
3781: It is worth stating the final result explicitly for
3782: a few simple cases cases.
3783: %
3784: \begin{corollary}
3785: %
3786: Consider communication over degraded, smooth and complete family
3787: $\{\BMS(\ent)\}_{\ent}$, using uniformly
3788: random codes from the ensemble $\eldpc n \ledge \redge$ and assume that the
3789: rate of this ensemble converges to the design rate.
3790: Assume that the $\BP$ fixed point family
3791: $\{\BMS(\ent),\Ldens{a}_{\ent}\}$, is smooth and complete.
3792: Then, for almost every $\ent\in [0,1]$
3793: %
3794: \begin{align*}
3795: %
3796: \lim_{\ell\to\infty}\lim_{n\to\infty}\expectation\,\Delta^{(\ell)}(Y) = 0\, .
3797: %
3798: \end{align*}
3799: %
3800: \end{corollary}
3801: %
3802: The proof follows easily from Corollary \ref{CoroCycle}.
3803:
3804: A somewhat more general statement is the following.
3805: %
3806: \begin{corollary}
3807: Consider communication over over degraded, smooth and complete family
3808: $\{\BMS(\ent)\}_{\ent}$, using uniformly
3809: random codes from the ensemble $\eldpc n \ledge \redge$ and assume that the
3810: rate of this ensemble converges to the design rate.
3811: Assume that the upper bound in on the $\MAP$ threshold in
3812: Theorem \ref{theo:UBMAP} is tight: $\ent^{\MAPsmall}= \overline{\ent}$.
3813: Then, for almost any $\ent\in [\overline{\ent},1]$,
3814: %
3815: \begin{align*}
3816: %
3817: \lim_{\ell\to\infty}\lim_{n\to\infty}\expectation\,\Delta^{(\ell)}(Y) = 0\, .
3818: %
3819: \end{align*}
3820: %
3821: \end{corollary}
3822: %
3823: \begin{proof}
3824: %
3825: Proceeding as in the proof of Theorem \ref{theo:UBMAP},
3826: one obtain that
3827: %
3828: \begin{align*}
3829: %
3830: \int_{\overline{\ent}}^1\, \gexitf {}(\ent) \, \de\ent=
3831: \int_{\overline{\ent}}^1\, \gexitf {\BPsmall}(\ent) \, \de\ent = r\, .
3832: %
3833: \end{align*}
3834: %
3835: Since $\gexitf {}(\ent)\le \gexitf {\BPsmall}(\ent)$ for all $\ent$,
3836: we have necessarily $\gexitf {}(\ent)= \gexitf {\BPsmall}(\ent)$
3837: for almost any $\ent\in[\overline{\ent},1]$. The thesis follows by
3838: applying Theorem \ref{thm:BPcorrect}.
3839: \end{proof}
3840: %
3841: %**************************************************************************
3842:
3843: \section{Why We Can Not Surpass Capacity: The Matching Condition}
3844: \label{sec:Matching}
3845:
3846: The upper bound $\overline{\ent}$ on the MAP threshold,
3847: cf. Theorem \ref{theo:UBMAP} cannot be larger than the
3848: Shannon threshold $1-r$. This follows by noticing that the $\gexit$
3849: kernel is not larger than $1$, and implies that iterative coding systems
3850: do not allow to communicate reliably above capacity.
3851: Of course, this result is also a straightforward consequence of Shannon's
3852: channel coding theorem. In this section we shall provide yet another
3853: proof of this basic fact. The interest of the new proof is three-fold:
3854: $(i)$ it does not assume communication over a smooth channel family;
3855: $(ii)$ it uses only quantities appearing in density evolution (and not
3856: just fixed points); $(iii)$ component codes (and their `matching')
3857: play a crucial role.
3858:
3859: For general BMS channels,
3860: and motivated by the
3861: geometric statement observed for the
3862: BEC and the relationship between the derivative of the
3863: mutual information and the MSE introduced by \cite{GSV04,GSV05},
3864: a similar chart,
3865: called MSE chart was constructed by Bhattad and Narayanan \cite{BaN04}.
3866: Assuming that the input densities to the component codes are Gaussian, this
3867: chart again fulfills the Area Theorem.
3868: In order to apply the MSE chart in the context of iterative coding
3869: the authors proposed to approximate the intermediate densities which
3870: appear in density evolution by ``equivalent'' Gaussian densities.
3871: This was an important first step in generalizing the matching condition
3872: to the whole class of BMS channels.
3873: In the following we show how to overcome the need for making the
3874: Gaussian approximation by using GEXIT functions.
3875:
3876: To start, let us review the case of transmission over the $\BEC(\ent)$
3877: using a degree distribution pair $(\ledge, \redge)$.
3878: In this case density evolution is equivalent to the EXIT chart
3879: approach and the condition for successful decoding under \BP\ reads
3880: \begin{align*}
3881: c(x) \defas 1-\redge(1-x) \leq \ledge^{-1}(x/\ent) \defas v^{-1}_{\ent}(x).
3882: \end{align*}
3883: This is shown in Fig.~\ref{fig:becmatching} for the degree distribution pair
3884: $(\ledge(x)=x^3, \redge(x)=x^4)$.
3885: \begin{figure}[hbt]
3886: \centering
3887: \setlength{\unitlength}{1.0bp}%
3888: \begin{picture}(110,110)
3889: \put(0,0){\includegraphics[scale=1.0]{exitchartbecldpcc}}
3890: %\put(100,-3){\makebox(0,0)[t]{\small $1$}}
3891: %\put(-5,102){\makebox(0,0){\small $1$}}
3892: \put(112, 0){\makebox(0,0)[l]{\small $x$}}
3893: %\put(18, 93){\makebox(0,0){\small $\ledge^{-1}(x/\ent)$}}
3894: \put(40, 60){\makebox(0,0){\small $c(x)$}}
3895: \put(16, 89){\makebox(0,0){\small $v^{-1}_{\ent}(x)$}}
3896: \put(58, 102){\makebox(0,0)[b]{{\small $\ent=0.58$}}}
3897: \put(100, -2){\makebox(0,0)[tr]{$h_{\text{out-variable}}=h_{\text{in-check}}$}}
3898: \put(-4,100){\makebox(0,0)[tr]{\rotatebox{90}{$h_{\text{out-check}}=h_{\text{in-variable
3899: }}$}}}
3900: \end{picture}
3901: \caption{\label{fig:becmatching} The EXIT chart method for
3902: the degree distribution $(\ledge(x)=x^3, \redge(x)=x^4)$ and
3903: transmission over the $\BEC(\ent = 0.58)$.}
3904: \end{figure}
3905: The area under the curve $c(x)$ equals $1-\int \!\redge$ and the
3906: area to the left of the curve $v^{-1}_{\ent}(x)$ is equal to
3907: $\ent \int \!\ledge$. By the previous remarks, a necessary condition
3908: for successful \BP\ decoding
3909: is that these two areas do not overlap.
3910: Since the total area equals $1$ we get the necessary condition
3911: $\ent \int \ledge+1-\int \redge\leq 1$. Rearranging terms, this
3912: is equivalent to the condition
3913: \begin{align*}
3914: 1-C_{\Shsmall} = \ent \leq \frac{\int \redge}{\int \ledge}= 1 - r(\ledge, \redge).
3915: \end{align*}
3916: In words, the design rate $r(\ledge, \redge)$ of any LDPC ensemble which, for
3917: increasing block lengths, allows successful
3918: decoding over the $\BEC(\ent)$, can not surpass the Shannon limit
3919: $1-\ent$.
3920: An argument very similar to the above was introduced
3921: by Shokrollahi and Oswald \cite{Sho00,OsS01} (albeit not using the language and geometric
3922: interpretation of EXIT functions and applying a slightly different range of integration).
3923: It was the first bound on the performance of iterative systems in which the Shannon capacity
3924: appeared explicitly using only quantities of density evolution.
3925: A substantially more general version of this bound can be found in
3926: \cite{AKtB02a,AKTB02,AKtB04}. The extension to parallel turbo schemes is addressed
3927: in \cite{MeU02jccc,MeU03a}. See also \cite{For05}.
3928:
3929: Although the final result (namely that transmission above capacity
3930: is not possible) is trivial, the method of proof is well worth the effort
3931: since it shows how capacity enters in the calculation of the performance
3932: of iterative coding systems. By turning this bound around, we
3933: can find conditions under which iterative systems achieve capacity:
3934: In particular it shows that the two component-wise
3935: EXIT curves have to be matched perfectly. Indeed, all currently known
3936: capacity achieving degree-distributions for the BEC can be derived
3937: by starting with this perfect matching condition and working backwards.
3938: Let us now show that, by using component-wise GEXIT functions, the perfect
3939: matching condition holds in the general case. This might in the future serve
3940: as a starting point to find capacity-achieving degree distributions for
3941: general BMS channels. We need one preliminary definition.
3942:
3943: \bdefi[Interpolating Channel Families]
3944: \label{def:interpolation}
3945: Consider a degree distribution pair $(\ledge, \redge)$
3946: and transmission over the BMS channel characterized by its
3947: $L$-density $\Ldens{c}$. Let $\Ldens{a}_{-1}=\Delta_0$
3948: and $\Ldens{a}_0=\Ldens{c}$ and set $\Ldens{a}_{\alpha}$,
3949: $\alpha \in [-1, 0]$, to
3950: $\Ldens{a}_{\alpha}=-\alpha \Ldens{a}_{-1} + (1+\alpha) \Ldens{a}_0$.
3951: The {\em interpolating density evolution families}
3952: $\{\Ldens{a}_{\alpha}\}_{\alpha=-1}^{\infty}$
3953: and $\{\Ldens{b}_{\alpha}\}_{\alpha=0}^{\infty}$ are then defined as follows:
3954: \begin{align*}
3955: \Ldens{b}_{\alpha} & = \sum_{i} \redge_i \Ldens{a}_{\alpha-1}^{\boxast (i-1)},
3956: \;\;\;\;\; \alpha \geq 0,\\
3957: \Ldens{a}_{\alpha} & =
3958: \sum_{i} \ledge_i \Ldens{c} \conv \Ldens{b}_{\alpha}^{\conv (i-1)},
3959: \;\;\;\;\;\alpha \geq 0,
3960: \end{align*}
3961: where $\conv$ denotes the standard convolution of densities and
3962: $\Ldens{a} \boxast \Ldens{b}$ denotes the density at the output of
3963: a check node, assuming that the input densities are $\Ldens{a}$ and $\Ldens{b}$,
3964: respectively.
3965: \edefi
3966: Discussion: First note that $\Ldens{a}_{\ell}$ ($\Ldens{b}_{\ell}$),
3967: $\ell \in \naturals$,
3968: represents the sequence of $L$-densities of density evolution
3969: emitted by the variable (check) nodes in the $\ell$-th iteration.
3970: By starting density evolution not only with $\Ldens{a}_{0}=\Ldens{c}$
3971: but with all possible convex combinations of $\Delta_0$ and
3972: $\Ldens{c}$, this discrete sequence of densities is completed to
3973: form a continuous family of densities ordered by physical degradation.
3974: The fact that the densities are ordered by physical degradation
3975: can be seen as follows: note that the computation tree for $\Ldens{a}_{\alpha}$
3976: can be constructed by taking
3977: the standard computation tree of $\Ldens{a}_{\lceil \alpha \rceil}$
3978: and independently erasing the observation associated to each variable leaf node with probability
3979: $\lceil \alpha \rceil-\alpha$. It follows that we can convert the computation tree of
3980: $\Ldens{a}_{\alpha}$ to that of $\Ldens{a}_{\alpha-1}$ by erasing all
3981: observations at the leaf nodes and by independently erasing
3982: each observation in the second (from the bottom) row of variable nodes
3983: with probability $\lceil \alpha \rceil-\alpha$.
3984: The same statement is true for $\Ldens{b}_{\alpha}$.
3985: If $\lim_{\ell \rightarrow \infty} \entropy(\Ldens{a}_{\ell})=0$, i.e.,
3986: if \BP\
3987: decoding is successful in the limit of large blocklengths, then
3988: the families are both complete.
3989: \begin{example}[Density Evolution and Interpolation]
3990: Consider transmission over the $\BSC(\epsilon = 0.07)$ using a
3991: $(3, 6)$-regular ensemble. Fig.~\ref{fig:debsc} depicts
3992: the density evolution process for this case.
3993: \begin{figure}[htp]
3994: \setlength{\unitlength}{0.5bp}%
3995: \begin{center}
3996: \begin{picture}(480,380)
3997: \put(0,210)
3998: {
3999: \put(0,0){\includegraphics[scale=0.5]{de1}}
4000: \put(120,50){\includegraphics[scale=0.5]{b1}}
4001: \put(0,120){\includegraphics[scale=0.5]{a0}}
4002: \put(50, 110){\makebox(0,0)[c]{\tiny $a_{0}$}}
4003: \put(170, 40){\makebox(0,0)[c]{\tiny $b_{1}$}}
4004: \put(50, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4005: \put(-2, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4006: }
4007: \put(260,210)
4008: {
4009: \put(0,0){\includegraphics[scale=0.5]{de2}}
4010: \put(120,50){\includegraphics[scale=0.5]{b2}}
4011: \put(0,120){\includegraphics[scale=0.5]{a1}}
4012: \put(50, 110){\makebox(0,0)[c]{\tiny $a_{1}$}}
4013: \put(170, 40){\makebox(0,0)[c]{\tiny $b_{2}$}}
4014: \put(50, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4015: \put(-2, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4016: }
4017: \put(0,0)
4018: {
4019: \put(0,0){\includegraphics[scale=0.5]{de3}}
4020: \put(120,50){\includegraphics[scale=0.5]{b3}}
4021: \put(0,120){\includegraphics[scale=0.5]{a2}}
4022: \put(50, 110){\makebox(0,0)[c]{\tiny $a_{2}$}}
4023: \put(170, 40){\makebox(0,0)[c]{\tiny $b_{3}$}}
4024: \put(50, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4025: \put(-2, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4026: }
4027: \put(260,0)
4028: {
4029: \put(0,0){\includegraphics[scale=0.5]{de25}}
4030: \put(120,50){\includegraphics[scale=0.5]{b13}}
4031: \put(0,120){\includegraphics[scale=0.5]{a12}}
4032: \put(50, 110){\makebox(0,0)[c]{\tiny $a_{12}$}}
4033: \put(170, 40){\makebox(0,0)[c]{\tiny $b_{13}$}}
4034: \put(50, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4035: \put(-2, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4036: }
4037: \end{picture}
4038: \end{center}
4039: \caption{\label{fig:debsc} Density evolution for $(3, 6)$-regular ensemble over $\BSC(0.07)$.}
4040: \end{figure}
4041: This process gives rise to the sequences of densities $\{\Ldens{a}_{\ell}\}_{\ell =0}^{\infty}$,
4042: and $\{ \Ldens{b}_{\ell}\}_{\ell=1}^{\infty}$. Fig.~\ref{fig:interpolation} shows
4043: the interpolation of these sequences for the choices $\alpha=1.0, 0.95, 0.9$ and $0.8$
4044: and the complete such family.
4045: \begin{figure}[htp]
4046: \setlength{\unitlength}{0.6bp}%
4047: \begin{center}
4048: \begin{picture}(360,250)
4049: \put(0, 150){
4050: \put(0,0){\includegraphics[scale=0.6]{de25}}
4051: \put(50, 102){\makebox(0,0)[b]{\tiny $\alpha=1.0$}}
4052: \put(50, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4053: \put(-2, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4054: \put(130,0){\includegraphics[scale=0.6]{de52}}
4055: \put(180, 102){\makebox(0,0)[b]{\tiny $\alpha=0.95$}}
4056: \put(180, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4057: \put(108, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4058: \put(260,0){\includegraphics[scale=0.6]{de53}}
4059: \put(310, 102){\makebox(0,0)[b]{\tiny $\alpha=0.9$}}
4060: \put(310, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4061: \put(258, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4062: }
4063: \put(-390,0)
4064: {
4065: \put(390,0){\includegraphics[scale=0.6]{de54}}
4066: \put(440, 102){\makebox(0,0)[b]{\tiny $\alpha=0.8$}}
4067: \put(440, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4068: \put(388, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4069: \put(520,0){\includegraphics[scale=0.6]{de55}}
4070: \put(570, -2){\makebox(0,0)[t]{\tiny $\entropy(\Ldens{a})$}}
4071: \put(518, 50){\makebox(0,0)[r]{\tiny \rotatebox{90}{$\entropy(\Ldens{b})$}}}
4072: }
4073: \end{picture}
4074: \end{center}
4075: \caption{\label{fig:interpolation} Interpolation of densities.}
4076: \end{figure}
4077: \end{example}
4078:
4079: \blemma
4080: Consider a degree distribution pair $(\ledge, \redge)$
4081: and transmission over an BMS channel characterized by its
4082: $L$-density $\Ldens{c}$ so that density evolution converges to
4083: $\Delta_{\infty}$.
4084: Let $\{\Ldens{a}_{\alpha}\}_{\alpha=-1}^{\infty}$
4085: and $\{\Ldens{b}_{\alpha}\}_{\alpha=0}^{\infty}$ denote the interpolated
4086: families as defined in Definition \ref{def:interpolation}.
4087:
4088: Then the two GEXIT curves parameterized by
4089: \begin{align*}
4090: \{ \entropy(\Ldens{a}_{\alpha}),
4091: \gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1}) \}, \tag*{GEXIT of check nodes} \\
4092: \{ \entropy(\Ldens{a}_{\alpha}),
4093: \gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha}) \}, \tag*{inverse of dual GEXIT of variable nodes}
4094: \end{align*}
4095: do not cross and faithfully represent density evolution.
4096: Further, the area under the ``check-node'' GEXIT function
4097: is equal to $1-\int \!\redge$ and the area to the left of the
4098: ``inverse dual variable node'' GEXIT function is equal to $\entropy(\Ldens{c}) \int \!\ledge$.
4099: It follows that $r(\ledge, \redge) \leq 1-\entropy(\Ldens{c})$, i.e.,
4100: the design rate can not exceed the Shannon limit.
4101: \elemma
4102: \begin{proof}
4103: First note that $\{ \entropy(\Ldens{a}_{\alpha}),
4104: \gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1}) \}$
4105: is the standard GEXIT curve representing the action
4106: of the check nodes: $\Ldens{a}_{\alpha}$ corresponds to
4107: the density of the messages {\em entering} the check nodes and
4108: $\Ldens{b}_{\alpha+1}$ represents the density of the corresponding
4109: output messages.
4110: On the other hand,
4111: $\{ \entropy(\Ldens{a}_{\alpha}),
4112: \gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha}) \}$
4113: is the inverse of the dual GEXIT curve
4114: corresponding to the action at the variable nodes:
4115: now the input density to the check nodes is
4116: $\Ldens{b}_{\alpha}$ and $\Ldens{a}_{\alpha}$ denotes the
4117: corresponding output density.
4118:
4119: The fact that the two curves do not cross can be seen as follows.
4120: Fix an entropy value. This entropy value corresponds to a
4121: density $\Ldens{a}_{\alpha}$ for a unique value of $\alpha$.
4122: The fact that
4123: $G(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha}) \geq
4124: G(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1})$ now follows from
4125: the fact that $\Ldens{b}_{\alpha+1} \prec \Ldens{b}_{\alpha}$ and
4126: that for any symmetric $\Ldens{a}_{\alpha}$ this relationship
4127: stays preserved by applying the GEXIT functional according to
4128: Corollary \ref{lemma:orderingviaphysicaldegradationexit}.
4129:
4130: The statements regarding the areas of the two curves
4131: follow in a straightforward manner from the GAT and Lemma \ref{lem:dualgexit}.
4132: The bound on the achievable rate follows in the same manner as for
4133: the BEC: the total area of the GEXIT box equals one and the two curves do not
4134: overlap and have areas $1-\int \redge$ and $\entropy(\Ldens{c})$.
4135: It follows that
4136: $1-\int \!\redge + \entropy(\Ldens{c}) \int \!\ledge \leq 1$,
4137: which is equivalent to the claim $r(\ledge, \redge) \leq 1-\entropy(\Ldens{c})$.
4138: \end{proof}
4139:
4140: We see that the matching condition still holds for general channels.
4141: There are a few important differences between the general case and the simple
4142: case of transmission over the BEC. For the BEC, the intermediate densities
4143: are always the BEC densities independent of the degree distribution.
4144: This of course enormously simplifies the task. Further, for the BEC, given
4145: the two EXIT curves, the progress of density evolution is simply given
4146: by a staircase function bounded by the two EXIT curves. For the general case,
4147: this staircase function still has vertical pieces but the ``horizontal''
4148: pieces have in general a non-vanishing slope. This is true since the $y$-axis for
4149: the ``check node'' step measures
4150: $\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1})$, but
4151: in the subsequent ``inverse variable node'' step
4152: it measures
4153: $\gentropy(\Ldens{a}_{\alpha+1}, \Ldens{b}_{\alpha+1})$.
4154: Therefore, one should think of two sets of labels on the $y$-axis,
4155: one measuring $\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1})$,
4156: and the second one measuring $\gentropy(\Ldens{a}_{\alpha+1}, \Ldens{b}_{\alpha+1})$. The ``horizontal'' step then consists of first
4157: switching from the first $y$-axis to the second, so that the labels
4158: correspond to the same density $\Ldens{b}$ and then drawing a horizontal
4159: line until it crosses the ``inverse variable node'' GEXIT curve.
4160: The ``vertical'' step stays as before, i.e., it really corresponds to
4161: drawing a vertical line. All this is certainly best clarified by
4162: a simple example.
4163: \bex[$(3, 6)$ Ensemble and Transmission over $\BSC$]
4164: Consider the $(3, 6)$-regular ensemble and transmission over the $\BSC(0.07)$.
4165: The corresponding illustrations are shown in Fig.~\ref{fig:componentgexit}.
4166: The top-left figure shows the standard GEXIT curve for the check node side.
4167: The top-right figure shows the dual GEXIT curve corresponding to the
4168: variable node side. In order to use these two curves in the same figure,
4169: it is convenient to consider the inverse function for the variable
4170: node side. This is shown in the bottom-left figure. In the bottom-right
4171: figure both curves are shown together with the ``staircase'' like function
4172: which represents density evolution. As we see, the two curves to not overlap
4173: and have both the correct areas.
4174: \begin{figure}[hbt]
4175: \centering
4176: \setlength{\unitlength}{1.0bp}
4177: \begin{picture}(230,230)
4178: \put(0,130){
4179: \put(0,0){\includegraphics[scale=1.0]{componentgexit1}}
4180: \put(50, -2){\makebox(0,0)[t]{\small $\entropy(\Ldens{a}_{\alpha})$}}
4181: \put(102, 50){\makebox(0,0)[l]{\small \rotatebox{90}{$\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1})$}}}
4182: \put(50, 40){\makebox(0,0)[t]{\small GEXIT: checks}}
4183: \put(50, 30){\makebox(0,0)[t]{\small $\text{area}=\frac56$}}
4184: \put(50, 10){\makebox(0,0)[c]{$\Ldens{b}_{\alpha+1} = \sum_{i} \redge_i \Ldens{a}_{\alpha}^{\boxast (i-1)} $}}
4185: }
4186: \put(130, 130)
4187: {
4188: \put(0,0){\includegraphics[scale=1.0]{componentgexit2}}
4189: \put(50, -2){\makebox(0,0)[t]{\small {$\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha})$}}}
4190: \put(-2, 50){\makebox(0,0)[r]{\small \rotatebox{90}{$\entropy(\Ldens{a}_{\alpha})$}}}
4191: \put(50, 70){\makebox(0,0)[t]{\small dual GEXIT: variables}}
4192: \put(50, 60){\makebox(0,0)[t]{\small $\text{area}=\frac13 h(0.07)$}}
4193: \put(102, 36.6){\makebox(0,0)[l]{\small \rotatebox{90}{$h(0.07) \approx 0.366$}}}
4194: \put(50, 40){\makebox(0,0)[c]{$\Ldens{a}_{\alpha} = \Ldens{c} \conv \sum_{i} \ledge_i \Ldens{b}_{\alpha}^{\conv (i-1)} $}}
4195: }
4196: \put(0,0)
4197: {
4198: \put(0,0){\includegraphics[scale=1.0]{componentgexit3}}
4199: \put(50, -2){\makebox(0,0)[t]{\small $\entropy(\Ldens{a}_{\alpha})$}}
4200: \put(-2, 50){\makebox(0,0)[r]{\small \rotatebox{90}{$\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha})$}}}
4201: \put(50, 30){\makebox(0,0)[t]{\small inverse of dual GEXIT:}}
4202: \put(50, 20){\makebox(0,0)[t]{\small variables}}
4203: \put(36.6, 102){\makebox(0,0)[b]{\small $h(0.07) \approx 0.366$}}
4204: }
4205: \put(130,0)
4206: {
4207: \put(0,0){\includegraphics[scale=1.0]{componentgexit4}}
4208: \put(50, -2){\makebox(0,0)[t]{\small $\entropy(\Ldens{a}_{\alpha})$}}
4209: \put(-2, 50){\makebox(0,0)[r]{\small \rotatebox{90}{$\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha})$}}}
4210: \put(102, 50){\makebox(0,0)[l]{\small \rotatebox{90}{$\gentropy(\Ldens{a}_{\alpha}, \Ldens{b}_{\alpha+1})$}}}
4211: \put(36.6, 102){\makebox(0,0)[b]{\small $h(0.07) \approx 0.366$}}
4212: }
4213: \end{picture}
4214: \caption{
4215: \label{fig:componentgexit}
4216: Faithful representation of density evolution by two non-overlapping component-wise
4217: GEXIT functions which represent the ``actions'' of the
4218: check nodes and variable nodes,
4219: respectively. The area between the two curves is proportional to the additive
4220: gap to capacity.
4221: }
4222: \end{figure}
4223: \eex
4224:
4225: As remarked earlier, one potential use of the matching condition
4226: is to find capacity approaching degree distribution pairs. Let us
4227: quickly outline a further such potential application. Assuming that
4228: we have found a sequence of capacity-achieving degree distributions,
4229: how does the number of required iterations scale as we approach capacity.
4230: It has been conjectured that the the number of required iterations
4231: scales like $1/\delta$, where $\delta$ is the gap to capacity.
4232: This conjecture is based on the geometric picture which the
4233: matching condition implies. To make things simple, imagine
4234: the two GEXIT curves as two parallel lines, lets say both
4235: at a 45 degree angle, a certain distance
4236: apart, and think of density evolution as a staircase function.
4237: From the previous results, the area between the lines is proportional
4238: to $\delta$. Therefore, if we half $\delta$ the distance between
4239: the lines has to be halved and one would expect that we need
4240: twice as many steps. Obviously, the above discussion was
4241: based on a number of simplifying assumptions. It remains to
4242: be seen if this conjecture can be proven rigorously.
4243: %
4244: %******************************************************************
4245: %
4246: \section{Conclusion}
4247: \label{sec:conclusion}
4248:
4249: Since the introduction of $\exit$ functions for the analysis
4250: iterative coding systems \cite{teB99a,teB99b,tBr00,teB00,teB01},
4251: researchers have tried to substantiate
4252: theoretically the empirical area rules that these seemed to satisfy.
4253: In this paper we showed how to {\em prove} these rules in a
4254: very general setting. The price to pay was to replace $\exit$
4255: functions by $\gexit$ functions. Fortunately,
4256: $\gexit$ functions are as simple to compute as ordinary $\exit$ functions
4257: and share in general many of their properties.
4258:
4259: We also presented several applications of this new tool.
4260: Most notably: $(i)$ It allows one to prove an upper bound on
4261: the $\MAP$ threshold which is conjectured to coincide with the
4262: actual threshold for several classes of ensembles (e.g. regular ones).
4263: $(ii)$ Via extended $\BP$ $\gexit$ curves, it provides some constraints
4264: on the relation between $\BP$ and $\MAP$ decoding. These constraints
4265: lead naturally to the Maxwell construction which provides the precise
4266: connection between the two. In particular we found that the $\BP$
4267: soft bit estimates are asymptotically exact for a noise range
4268: {\em above threshold}.
4269: $(iii)$ It implies a matching constraint on component codes of
4270: capacity-achieving systems.
4271:
4272: These results open many research directions. It may be worth to list a
4273: few of them.
4274:
4275: Prove existence, uniqueness and
4276: regularity properties of asymptotic $\MAP$ and extended $\BP$
4277: $\gexit$ curves. In particular, we expect that the last one is a
4278: smooth single valued function of the entropy of the fixed point density.
4279: The iterative procedure which we presented in Section \ref{sec:HowToEBP}
4280: only {\em proves} that
4281: for each message entropy there is at least one fixed point of density
4282: evolution. But, empirically, when running this algorithm,
4283: we found that indeed there seems to be a unique such fixed point
4284: and that all these fixed points seem to form a smooth manifold.
4285: Further,
4286: we proved several partial results in
4287: this direction (for instance existence for $\EBP$ curves, uniqueness
4288: for $\MAP$ curves, etc). However, the general question remains open.
4289:
4290: Prove that the Maxwell construction indeed provides the correct connection
4291: between $\MAP$ and $\BP$ $\gexit$ curves. As particular case
4292: (which may well be simpler than the general statement),
4293: prove the upper bound (\ref{theo:UBMAP}) is
4294: indeed tight for some selected ensembles, e.g. for regular ones.
4295:
4296: Use the interpolation construction of Section~\ref{sec:Matching}
4297: to prove a lower bound on the number of message passing iterations as a
4298: function of the gap to capacity.
4299: %
4300: %******************************************************************
4301: %
4302: \section*{Acknowledgment}
4303: The authors would like to thank Nicolas Macris
4304: and Olivier L\'ev\^eque for useful discussions.
4305:
4306: A.M. has been partially supported by the EU integrated project EVERGROW.
4307: %
4308: %******************************************************************
4309: %
4310: \appendices
4311:
4312: \section{GEXIT Kernel over Gaussian Channels}
4313: \label{app:gaussiankernel} \label{app:kernellimits} \label{sec:mmse}
4314:
4315: This appendix contains a few useful results concerning the
4316: $\gexit$ kernel for Gaussian channels.
4317:
4318: \blemma[$\gexit$ Kernel, $L$-Domain -- $\{\text{BAWGNC}(\ent)\}$]\label{lemma:equivkernel}
4319: Consider the family
4320: $\{\Ldens{c}_{\BAWGNCsmall(\ent)}\}$ of BAWGN channels, where $\ent$ denotes
4321: the channel entropy. The channel model
4322: is therefore $Y=X+N$, where $X$ takes values $x\in{\cal{X}}=\{-1,+1\}$ and
4323: $N$ is Gaussian with zero mean and variance $\sigma^2$.
4324: Then the following represent {\em equivalent} kernels:
4325: %
4326: \begin{align*}
4327: %
4328: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z
4329: &= \displaystyle
4330: \renewcommand{\cp}{\sigma}
4331: \frac{ \text{e}^{-z}\int_{-\infty}^{+\infty}\scriptstyle
4332: \frac{ \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}} %\text{e}^{-\frac{(w-\cp)^2}{4 \cp}}
4333: }{(\cosh(\frac{w-z}{2}))^2}\text{d}w } {\int_{-\infty}^{+\infty}\scriptstyle
4334: \frac{ \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}}
4335: }{(\cosh(\frac{w}{2}))^2} \text{d}w}, \tag*{(i)}\\
4336: \gexitkl{' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z
4337: &= \renewcommand{\cp}{\sigma}
4338: \frac{1-\expectation[\expectation[X|Y,\Phi=z]^2]}{1-\expectation[\expectation[X|Y]^2]},
4339: \tag*{(ii)}\\
4340: \gexitkl{'' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z
4341: & = \renewcommand{\cp}{\sigma}
4342: \frac{1-\expectation[X|Y,\Phi=z]}{1-\expectation[X|Y]}. \tag*{(iii)}
4343: \renewcommand{\cp}{\epsilon}
4344: %
4345: \end{align*}
4346: %
4347: Hereby, $\Phi$ denotes a further observation of $X$ which is conditionally
4348: independent of $Y$, which is the result of passing $X$ through
4349: a symmetric channel, and which is assumed to
4350: be in log-likelihood form (if we use coding,
4351: $\Phi$ represents the extrinsic estimate
4352: of $X$ in the $L$-domain).
4353: \elemma
4354:
4355: Discussion: This lemma provides several equivalent representations
4356: of the kernel for the BAWGN channel. The expression (ii) shows the
4357: relationship between conditional entropy and mean-square error
4358: (MSE) estimator. To see this, observe first that the denominator is
4359: a ($z$ independent) scaling factor depending on our parameterization of
4360: the channel through its entropy $\ent$. Second, observe that
4361: the numerator $1-\expectation[\expectation[X|Y,\Phi=z]^2]=
4362: \expectation[\expectation[X^2|Y,\Phi=z]-\expectation[X|Y,\Phi=z]^2]$
4363: is the mean-square error estimator (which in this framework
4364: includes the decoding estimate $z$).
4365: This elegant relationship
4366: which connects a fundamental information theoretic quantity
4367: (the conditional entropy, or, equivalently, the mutual information)
4368: to a measure widely-used in signal processing was first observed
4369: by Guo, Shamai and Verd{\'u} in
4370: \cite{GSV04,GSV05}. In the above lemma, the channel
4371: inputs are binary. In Lemma \ref{lemma:gexitawgnEx} we give an alternative way
4372: of deriving $\gexitkl{{\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z$
4373: in the more general context of non-binary channel inputs.
4374:
4375: The form (iii) provides a further simplification. This expression, in
4376: which the numerator shows the magnetization was first stated
4377: in \cite{Mac05} using the Nishimori identity (in the context of coding,
4378: this identity was first discussed in \cite{Mon04}).
4379:
4380: Before proving Lemma \ref{lemma:equivkernel},
4381: let us recall the following well-known fact
4382: which will be used several times in the following:
4383: Consider a BMS channel $p_{Y|X}(y|x)$ and $f(y)$, a measurable function.
4384: If $f(y)$ is even, then
4385: $\expectation_Y [f(Y)]=\expectation_{Y|X=1} [f(Y)].$
4386: \bproof
4387: Under the all-one assumption, the channel
4388: density is
4389: $\Ldens{c}(w)\defas {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}(w) =\frac{\sigma}{\sqrt{8\pi}}\text{e}^{-\frac{(w\sigma^2-2)^2}{8\sigma^2}}$. \\
4390: (i) The kernel as stated in Lemma \ref{lem:gexitlinear} is expressed in
4391: terms of the derivative of $\Ldens{c}(w)$
4392: with respect to the channel parameter.
4393: To get a more pleasing analytic expression we use the fact that
4394: for the Gaussian case we can express this derivative via the
4395: %Choose the parameterization $\cp\defas2/\sigma^2$ and use
4396: identity
4397: $\frac{\partial\Ldens{c}(w)}{\partial \cp}=-\frac{\partial\Ldens{c}(w)}{\partial w}+
4398: \frac{\partial^2\Ldens{c}(w)}{\partial w^2}$.
4399: Now, use the parameterization
4400: $\cp\defas2/\sigma^2$.
4401: %For practical computations,
4402: % the derivative is first taken
4403: %with respect to a parameter $\cp$. This expression can further be normalized
4404: % (by the derivative of
4405: %$\ent=\int_{-\infty}^{+\infty}\Ldens{c}(w) \logtwo(1+\text{e}^{-w})\text{d}w$
4406: %with respect to $\cp$)
4407: %in order to get a derivative of $\Ldens{c}(w)$ with respect to $\ent$.
4408: %
4409: %A convenient parameterization for this case is $\cp\defas2/\sigma^2$.
4410: %After some steps of calculus, and using the identity
4411: %$\frac{\partial\Ldens{c}(w)}{\partial \cp}=-\frac{\partial\Ldens{c}(w)}{\partial w}+\frac{\partial^2\Ldens{c}(w)}{\partial w^2}$, we get
4412: Then using twice integration by parts (as in \cite{Mac05}), we get
4413: \begin{align*}
4414: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z \frac{\text{d}\epsilon}{\text{d}\ent}
4415: &= \int_{-\infty}^{+\infty}\frac{\partial\Ldens{c}(w)}{\partial \cp} \log(1+\text{e}^{-w-z})\text{d}w\\
4416: &=\int_{-\infty}^{+\infty}\frac{\partial\Ldens{c}(w)}{\partial w}\frac{\text{e}^{-w-z}}{1+\text{e}^{-w-z}}\text{d}w\\
4417: &~~~~ -\int_{-\infty}^{+\infty}\Ldens{c}(w)\frac{\text{e}^{-w-z}}{1+\text{e}^{-w-z}}\text{d}w\\
4418: & = \int_{-\infty}^{+\infty}\Ldens{c}(w)\frac{-1}{ (1+\text{e}^{w+z})^2}\text{d}w\\
4419: & = \frac{- \text{e}^{-z}}{4}\int_{-\infty}^{+\infty}\frac{\Ldens{c}(-w)}{(\cosh(\frac{w+z}{2}))^2}\text{d}w.
4420: \end{align*}
4421: The computation of $\frac{\text{d}\epsilon}{\text{d}\ent}$ is exactly the
4422: same if we set $z=0$.
4423: Therefore,
4424: \begin{align*}
4425: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z
4426: & \defas \frac{ \text{e}^{-z}\int_{-\infty}^{+\infty}\frac{ \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{(\cosh(\frac{w-z}{2}))^2}\text{d}w } {\int_{-\infty}^{+\infty}\frac{ \text{e}^{-\frac{(w-\cp)^2}{4 \cp}} }{(\cosh(\frac{w}{2}))^2} \text{d}w }.
4427: \end{align*}
4428:
4429: (ii) First, we claim that the previous expression can be written as
4430: \begin{align*}
4431: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z
4432: & = \text{e}^{-z}\frac{1-\expectation[\expectation[X|Y,\Phi=-z]^2]}{1-\expectation[\expectation[X|Y]^2]}.
4433: \end{align*}
4434: To see this, observe that
4435: \begin{align*}
4436: w+z & \overset{(a)}{=} \log \frac{p_{\frac{2Y}{\sigma^2}|X}(w|+1)}{p_{\frac{2Y}{\sigma^2}|X}(w|-1)} + \log \frac{p_{\Phi|X}(z|+1)}{p_{\Phi|X}(z|-1)}\\
4437: & \overset{(b)}{=} \log \frac{p_{\frac{2Y}{\sigma^2},\Phi|X}(w,z|+1)}{p_{\frac{2Y}{\sigma^2},\Phi|X}(w,z|-1)} \\
4438: & \overset{(c)}{=} \log \frac{ p_{X|\frac{2Y}{\sigma^2},\Phi}(+1|w,z) }{ p_{X|\frac{2Y}{\sigma^2},\Phi}(-1|w,z) },
4439: \end{align*}
4440: where $(a)$ comes from the definition of $w$ and $z$ in Lemma \ref{lemma:equivkernel},
4441: $(b)$ from the independence of $Y$ and $\Phi$ when $X$ is given, and where $(c)$ is the
4442: Bayes rule using $p_X(+1)=p_X(-1)=\frac12$. Therefore,
4443: \begin{align*}
4444: \tanh(\frac{w+z}{2}) & = \frac{1-\text{e}^{-w-z}}{1+\text{e}^{-w-z}} \\
4445: & = \frac{p_{X|\frac{2Y}{\sigma^2},\Phi}(+1|w,z) - p_{X|\frac{2Y}{\sigma^2},\Phi}(-1|w,z) }{p_{X|\frac{2Y}{\sigma^2},\Phi}(+1|w,z) + p_{X|\frac{2Y}{\sigma^2},\Phi}(-1|w,z) }\\
4446: & = \expectation[X|w,z].
4447: \end{align*}
4448: This quantity (which is often called ``soft bit'' as in \cite{HOP96}) is
4449: a bit estimate in the $D$-domain and the relationship
4450: $\expectation[X|w,z]=\tanh(\frac{w+z}{2})$ is in fact well-known.
4451: %Finally, we have used $(\tanh(\frac{w+z}{2}))^2=1-\frac{1}{(\cosh(\frac{w+z}{2}))^2}$.
4452: Therefore, since $1-(\tanh(\frac{w+z}{2}))^2=\frac{1}{(\cosh(\frac{w+z}{2}))^2}$,
4453: \begin{align*}
4454: \gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z
4455: & = \text{e}^{-z}\frac{1-\int_{-\infty}^{\infty}\Ldens{c}(w)(\tanh(\frac{w+z}{2}))^2\text{d}w}{1-\int_{-\infty}^{\infty}\Ldens{c}(w)(\tanh(\frac{w}{2}))^2\text{d}w}\\
4456: & \text{e}^{-z}\frac{1-\expectation_{Y|X=1}[(\tanh(\frac{Y+z}{2}))^2]}{1-\expectation_{Y|X=1}[(\tanh(\frac{Y}{2}))^2]},
4457: \end{align*}
4458: and the claim follows since, as discussed above, we can
4459: drop in the last expression the conditioning on $X=1$.
4460:
4461: Second, as discussed in Example \ref{ex:nonuniquekernel},
4462: the kernel is in general not unique in the $L$-domain
4463: and we can use this degree of freedom to get alternative kernels.
4464: Denote $f(z)\defas\frac{1-\expectation[\expectation[X_1|Y_1,-z]^2]}{1-\expectation[\expectation[X_1|Y_1]^2]}$ and observe that
4465: $\gexitkl {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} z=\exp(-z) f(z)$ with this notation. Then, for any symmetric density
4466: $\Ldens{a}(z)$, the function $\gexitkl{' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z \defas f(-z)$ is also
4467: a valid kernel for the $L$-domain since $\int_{-\infty}^{+\infty}\Ldens{a}(z) \text{e}^{-z}f(z)\text{d}z= \int_{-\infty}^{+\infty}\Ldens{a}(z) f(-z)\text{d}z$. Therefore, an alternative kernel is
4468: \begin{align*}
4469: \gexitkl{' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z
4470: &= \renewcommand{\cp}{\sigma}
4471: \frac{1-\expectation[\expectation[X|Y,z]^2]}{1-\expectation[\expectation[X|Y]^2]}
4472: =\frac{ \int_{-\infty}^{+\infty}\scriptstyle
4473: \frac{ % \text{e}^{-\frac{(w-\cp)^2}{4 \cp}}
4474: \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}}
4475: }{(\cosh(\frac{w+z}{2}))^2}\text{d}w } {\int_{-\infty}^{+\infty}\scriptstyle
4476: \frac{
4477: %\text{e}^{-\frac{(w-\cp)^2}{4 \cp}}
4478: \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}}
4479: }{(\cosh(\frac{w}{2}))^2} \text{d}w }.
4480: \end{align*}
4481:
4482:
4483: (iii) For any symmetric random variable $L$, a straightforward exercise shows that
4484: $\expectation[\tanh(L/2)]=\expectation[(\tanh(L/2))^2]$. See, e.g, \cite{Mac05,Mon04}.
4485: Applied to the symmetric random variable $L\defas\log \frac{p(Y|+1)}{p(Y|-1)}=\frac{2}{\sigma^2}Y$
4486: under the all-one assumption, this gives us
4487: $\expectation[\expectation[X|Y]^2]=\expectation[\tanh(\frac{L}{2})^2]=\expectation[\tanh(\frac{L}{2})]=\expectation[X|Y]$.
4488: Therefore the denominator can be easily written as
4489: $\frac{1}{1-\expectation[\expectation[X|Y]^2]}=\frac{1}{1-\expectation[X|Y]}$.
4490: We can not use directly this argument for the term $\expectation[\expectation[X|Y,z]^2]=\expectation[\tanh(\frac{Y}{\sigma^2}+\frac{z}{2})^2]$ at the numerator (the random variable $\frac{2}{\sigma^2}Y+z$ being not symmetric).
4491: However, we can
4492: look
4493: for an equivalent kernel.
4494: This is easily done by observing that the values $z$ are provided by the symmetric random
4495: variable $\Phi_{}$. The sum of two symmetric random variables is again symmetric,
4496: therefore $\frac{2}{\sigma^2}Y+\Phi_{}$ is symmetric. See, e.g., \cite{RiU05}.
4497: We can then use the fact that $\expectation[\tanh(L/2)]=\expectation[(\tanh(L/2))^2]$
4498: with $L\defas \frac{2}{\sigma^2}Y+\Phi_{}$
4499: to write $\expectation_{Y,\Phi_{}}[\expectation_{X}[X|Y,\Phi_{}]^2]=\expectation_{Y,\Phi_{}}[\expectation_{X}[X|Y,\Phi_{}]]$. Therefore,
4500: %
4501: \begin{align*}
4502: \gexitkl{'' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z
4503: & = \renewcommand{\cp}{\sigma}
4504: \frac{1-\expectation[X|Y,z]}{1-\expectation[X|Y]}
4505: =
4506: \frac{ \int_{-\infty}^{+\infty}
4507: \scriptstyle
4508: \frac{%2
4509: % \text{e}^{-\frac{(w-\cp)^2}{4 \cp}}
4510: \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}}
4511: }{1+\text{e}^{\scriptscriptstyle w+z}} \text{d}w }{\int_{-\infty}^{+\infty}\scriptstyle
4512: \frac{%2
4513: \text{e}^{-\frac{(w \cp^2-2)^2}{8 \cp^2}}
4514: }{1+\text{e}^{w}} \text{d}w} %\tag*{(vi)}
4515: \renewcommand{\cp}{\epsilon}
4516: \end{align*}
4517: %
4518: %
4519: is an equivalent kernel (but pointwise different from
4520: $\gexitkl{{\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z$
4521: and $\gexitkl{' {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}}} z$). The last equality comes from the fact that $1-\expectation[X|Y,z]=1-\expectation_{Y}[\tanh(\frac{Y+z}2)]=\expectation_{Y}[\frac{2}{1+\text{e}^{Y+z}}]$. %Therefore,
4522: \eproof
4523:
4524:
4525:
4526: GEXIT and EXIT curves are in general very similar. Next lemma illuminates this fact: it shows
4527: that, in the limit of small SNR, the kernel for the BAWGNC behaves similarly to the kernel
4528: for the BSC discussed in Example \ref{ex:lbsckernel}.
4529: \blemma[Limiting Behavior of GEXIT Kernel] Consider the family
4530: $\{\Ldens{c}_{\BAWGNCsmall(\ent)}\}$ of BAWGN channels, where $\ent$ denotes
4531: the channel entropy: The additive noise $N$ in the model $Y=X+N$ is Gaussian
4532: with zero-mean and variance $\sigma^2$.
4533: Then
4534: \begin{align*}
4535: \lim_{\sigma\to\infty}\gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} s & = 1-s^2,\tag*{(i)}\\
4536: \lim_{\sigma\to0}\gexitkabsd {\Ldens{c}_{\text{\tiny BAWGNC}(\ent)}} s & = 1.\tag*{(ii)}
4537: \end{align*}
4538: In the $|D|$-domain, the kernels are ordered between those two extremal functions.
4539: \elemma
4540: \bproof
4541: First recall the transform formula (\ref{equ:gexitkernelconversion})
4542: and $2\tanh^{-1}(s)=\log\frac{1+s}{1-s}$.
4543: (i) With expression (iii) of Lemma \ref{lemma:equivkernel}
4544: we have
4545: $\gexitkl{\Ldens{c}}{2\tanh^{-1}(s)}=\frac{1-\int_{-\infty}^{+\infty}\Ldens{c}(l)\tanh(l/2+\tanh^{-1}(s)) \text{d} l }{1-\int_{-\infty}^{+\infty}\Ldens{c}(l)\tanh(l/2) \text{d} l }$. Let us restrict ourself to the study of the term $I_\sigma(s)\defas\int_{-\infty}^{+\infty}\Ldens{c}(l)\tanh(l/2+\tanh^{-1}(s)) \text{d} l$. When $\sigma^2\to\infty$, then the distribution of the channel inputs (more exactly of the LLR's in the L-domain) $\Ldens{c}(l) =\frac{\sigma}{2\sqrt{2\pi}}\exp(-\frac{\sigma^2(l-2/\sigma^2)^2}{8})$ becomes a Dirac centered in 0 (since its variance $4/\sigma^2\to0$). For any function continuous in 0, e.g., for the function $k_s:l\mapsto \tanh(l/2+\tanh^{-1}(s)) $, one can indeed replace, without committing much error when $\sigma^2\to\infty$, the integral
4546: $\int_{-\infty}^{+\infty}\Ldens{c}(l) k_s(l) \text{d}l$ by
4547: $
4548: \int_{-\infty}^{+\infty}\Ldens{c}(l) k_s(0) \text{d}l.
4549: $
4550: See, e.g., \cite{Zem65} for further details. Therefore
4551: $$
4552: I_\sigma(z)\underset{\sigma\to\infty}{\longrightarrow}\tanh(0/2+\tanh^{-1}(s))=s.
4553: $$
4554: Using (\ref{equ:gexitkernelconversion}),
4555: we finally get $$\gexitkabsd{\Ldens{c}}{s}=\frac{1-s}{2}\frac{1+s}{1}+\frac{1+s}{2}\frac{1-s}{1}=1-s^2.$$
4556: (ii) The case $\sigma \to0$ corresponds to the full knowledge of
4557: the channel input. The kernel in the $|D|$-domain converges point-wise
4558: to 1. As used for density evolution, see \cite{RiU05}, in this case $\Ldens{c}(l)$ becomes a Dirac
4559: at $\infty$ a a similar argument as for (i) can be applied.
4560:
4561: For $\cp\in(0,1)$, the kernels in the $|D|-$domain are ordered because of
4562: Lemma \ref{lemma:orderingviaphysicaldegradationexit}.
4563: \eproof
4564:
4565: As discussed before Lemma \ref{lemma:equivkernel},
4566: the pleasing relationship presented
4567: in \cite{Guo04,GSV05} or \cite{Mac05} emerges for the BAWGNC.
4568: So far we have restricted ourself
4569: to the case of binary inputs. But the non-binary case
4570: as discussed in \cite{Guo04,GSV05,GSV05i} is not much harder. This
4571: is presented in Lemma \ref{lemma:gexitawgnEx} using our framework.
4572:
4573: \blemma[AWGN($\ent$)] \label{lemma:gexitawgnEx}
4574: Consider a length $\n$ code, call it $\graph$.
4575: Assume transmission takes place over a family $\{AWGNC(\ent_i)\}_{i\in[\n]}$ where
4576: there is a global parameter $\cp$ such that $\ent_i(\cp)=\ent(\cp)$ is the entropy associated to the $i^\text{th}$
4577: channel for all $i\in[\n]$. Let this
4578: parameter be
4579: $\cp=-2\snr\defas-\frac{2}{\sigma^2}$. Then
4580: $$
4581: \gexitfi{}(\graph,\cp)= \expectation \left[ \expectation[X_i^2|Y]-\expectation[X_i|Y]^2 \right].
4582: $$
4583: In words, the derivative of the conditional entropy with respect to the particular paramater $\cp$
4584: is equal to the Mean-Square Error (MSE) estimator.
4585: \elemma
4586:
4587: \begin{proof} We will
4588: prove the result in general settings when the input alphabet ${\cal{X}}$
4589: can be any subset of $\R$. Temporarily, let us denote $\tilde{Y}=X+\tilde{N}$
4590: our running Gaussian channel model. $\tilde{N}$ is the additive white Gaussian
4591: noise with zero-mean and variance $\sigma^2$. Now
4592: let us normalize this model by $\sigma^2$ to
4593: get the equivalent model $Y=\sqrt{\snr}X+N$ where $\snr=\frac{1}{\sigma^2}$
4594: and $N$ is an additive white Gaussian noise with zero-mean and unit-variance.
4595: In order to be a sufficient statistics,
4596: the extrinsic MAP estimate $\phi^{}_i=\phi^{}_i(y_{\sim i})$ can no longer
4597: be a log-likelihood ratio but, in general, a function of $x_i$, i.e., $\phi^{}_i:x\mapsto\phi^{}_i(y_{\sim i},x)$.
4598: From (\ref{equ:gexitcompact}), it follows that
4599: \begin{align*}
4600: \gexitfi {}(\graph,\cp) & =
4601: \int_{\phi^{}_i,y_i, x_i}
4602: p(x_i)
4603: p(\phi^{}_i \mid x_i)
4604: \frac{\text{d}\phantom{\ent}}{\text{d} \cp} p(y_i|x_i)\cdot \\
4605: &
4606: ~~~~~\cdot \log\left(\int_{x'_i}\frac{p(x'_i|\phi^{}_i)
4607: p(y_i|x'_i)}{p(x_i|\phi^{}_i)p(y_i|x_i)}\text{d}x_i'
4608: \right) \text{d}x_i \text{d} y_i \text{d} \phi_i^{}.
4609: \end{align*}
4610: To simplify the computations, a few remarks are of order.
4611: First recall that we have chosen $\cp$ to be $\cp=-2\snr=\frac{-2}{\sigma^2}$.
4612: Second, observe that the Gaussian density permits us to write
4613: $\frac{\text{d} p(y_i|x_i)}{\text{d} \cp} = \frac{x_i}{\sqrt{\snr}}\frac{\text{d}}{\text{d}y_i}p(y_i|x_i).$
4614: Therefore, integrating by parts with respect to $y_i$, we get
4615: \begin{align*}
4616: &~~~\gexitfi {}(\graph,\cp) \\
4617: & = \int_{\phi^{}_i,y_i, x_i}
4618: p(x_i)p(\phi^{}_i \mid x_i) \frac{x_i}{\sqrt{\snr}}
4619: p(y_i|x_i)\cdot \\
4620: &
4621: ~~~~~\cdot \frac{\text{d}}{\text{d}y_i}\left\{\log\left(\int_{x'_i}\frac{p(x'_i|\phi^{}_i)
4622: p(y_i|x'_i) }{p(x_i|\phi^{}_i)p(y_i|x_i)}\text{d}x_i'
4623: \right) \right\} \text{d}x_i \text{d} y_i \text{d} \phi_i^{}\\
4624: & =
4625: -\int_{\phi^{}_i,y_i, x_i}
4626: p(x_i)p(\phi^{}_i \mid x_i) \frac{x_i}{\sqrt{\snr}}
4627: p(y_i|x_i)\cdot \\
4628: &
4629: ~~~~~\cdot \frac{\int_{x_i'}\ \sqrt{\snr}(x_i'-x_i)p(x_i'|\phi^{}_i)p(y_i|x_i')
4630: \text{d}x_i'
4631: }{\int_{x'_i} p(x'_i|\phi^{}_i)
4632: p(y_i|x'_i) \text{d}x_i' }
4633: \text{d}x_i \text{d} y_i \text{d} \phi_i^{},
4634: \end{align*}
4635: after having used $\frac{\text{d}p(y_i|x_i')}{\text{d}y_i}=\frac{\text{d}p_{Z_i}(y_i-\sqrt{\snr}x_i')}{\text{d}y_i}=-(y_i-\sqrt{\snr}x_i')p(y_i|x_i')$.
4636: Let us now re-order as $p(x_i'|\phi_i)p(y_i|x_i')=p(x_i'|\phi_i,y_i)p(y_i|\phi_i)$ and use (with a slight abuse of notations) $\frac{y_i+\phi_i}{\sqrt{\snr}}=\expectation_{X_i}\left[X_i|\phi^{}_i,y_i\right]$ to get
4637: \begin{align*}
4638: & ~~~\gexitfi {}(\graph,\cp) \\
4639: & = -\int_{\phi^{}_i,y_i, x_i}
4640: p(x_i)p(\phi^{}_i \mid x_i) x_i
4641: p(y_i|x_i)\cdot \\
4642: & ~~~~~
4643: \cdot \frac{p(y_i|\phi_i) (\frac{(y_i+\phi_i) }{\sqrt{\snr}}-x_i) }{p(y_i|\phi_i)}
4644: \text{d}x_i \text{d} y_i \text{d} \phi_i^{}\\
4645: & = \int_{\phi^{}_i,y_i} p(\phi_i,y_i)\\
4646: & ~~~~~
4647: \cdot \int_{x_i} p(x_i|y_i,\phi_i) \left(x_i^2-\frac{(y_i+\phi_i) x_i}{\sqrt{\snr}}\right)\text{d}x_i \text{d} y_i \text{d} \phi_i^{}\\
4648: & = \int_{\phi^{}_i,y_i} p(\phi_i,y_i) \\
4649: & ~~~~~
4650: \cdot \left(\expectation_{X_i}\left[X_i^2|\phi^{}_i,y_i\right] - \expectation_{X_i}\left[X_i|\phi^{}_i,y_i\right]^2\right) \text{d} y_i \text{d} \phi_i^{}.
4651: \end{align*}
4652: This concludes our proof since $\Phi_i$ is a sufficient statistic for $Y_{\sim i}$.
4653: \end{proof}
4654:
4655: %In hindsight, it is interesting to note that in probability theory
4656: %a relationship which is equivalent to
4657: %the connection between the derivative of the mutual information
4658: %and the MSE is know under the name ``???'' and is due to
4659: %RussianGuy \cite{}.
4660:
4661: %************************************************************
4662: %
4663: \section{Physical Degradation: a Calculus Proof}
4664: \label{sec:AlternativePhysicalDegradation}
4665:
4666: In this appendix we provide a direct calculus proof of
4667: Corollary \ref{lemma:orderingviaphysicaldegradationexit}, exploiting
4668: the explicit representation provided by Lemma \ref{lem:gexitlinear}.
4669: As a byproduct we show that the $\gexit$ kernel in the
4670: $|D|$-domain is non-increasing and concave. This fact is also used in the
4671: proof of Lemma \ref{lem:GeneralBounds}.
4672:
4673: For our purpose it is convenient to represent all quantities
4674: in the $|D|$-domain.
4675: Let $\{\absDdens{c_{\BMSsmall(\ent)}}\}_{\ent}$ denote the
4676: family of $|D|$-densities characterizing the channel family.
4677: Let
4678: $\gexitkabsd {\BMSsmall(\ent)} w$ denote the $\gexit$ kernel
4679: in the $|D|$-domain as introduced in (\ref{equ:gexitkernelconversion}).
4680: We can rewrite it in the form
4681: \begin{align*}
4682: \gexitkabsd {\BMSsmall(\ent)} w & =
4683: \int_{0}^{1}
4684: \frac{\partial \absDdens{c_{\BMSsmall(\ent)}}(z)}{\partial \ent} \alpha(z, w) \mbox{d}z,
4685: \end{align*}
4686: where
4687: \begin{align*}
4688: \alpha(z, w) = \frac{1}{4} \sum_{i, j = \pm 1} (1+i z)(1+j w) \beta(i z, j w),
4689: \end{align*}
4690: with $\beta(z, w)=\log_2\bigl(1+e^{-2 \tanh^{-1}(z)} e^{-2 \tanh^{-1}(w)} \bigr)$.
4691: Finally, let $\absDdens{a}$ and $\absDdens{b}$ denote the two symmetric densities
4692: in the $|D|$-domain.
4693:
4694: The claim of the theorem is then equivalent to the statement
4695: that the $\gexit$ functional
4696: $\int_{0}^{1} \gexitkabsd {\BMSsmall(\ent)} w \absDdens{a}(w) \text{d} w$
4697: preserves the partial order
4698: implied by physical degradation. This means that if
4699: $\absDdens{a} \prec \absDdens{b}$ then
4700: \begin{align*}
4701: \int_{0}^{1} \gexitkabsd {\BMSsmall(\ent)} w \absDdens{a}(w) \text{d} w
4702: & \leq
4703: \int_{0}^{1} \gexitkabsd {\BMSsmall(\ent)} w \absDdens{b}(w) \text{d} w.
4704: \end{align*}
4705: By Theorem 3.4 in \cite{RiU05}, a $|D|$-domain kernel preserves the partial order
4706: implied by physical degradation if it is non-increasing and concave on $[0, 1]$,
4707: i.e., if its first two derivatives are non-positive. This means we need to show that
4708: \begin{align*}
4709: \int_{0}^{1}
4710: \frac{\partial \absDdens{c_{\BMSsmall(\ent)}}(z)}{\partial \ent} \frac{\partial^i \alpha(z, w)}{\partial w^i} \mbox{d}z \leq 0,
4711: \end{align*}
4712: for $i=1, 2$.
4713: By the same Theorem 3.4 the above condition is verified if both
4714: $\frac{\partial^i \alpha(z, w)}{\partial w^i}$ for $i=1, 2$, are convex and
4715: non-decreasing. This in turn is true if
4716: $\frac{\partial^{i+j} \alpha(z, w)}{\partial w^i \partial z^j} \geq 0$ for
4717: $i, j =1, 2$. Now some further calculus shows that
4718: \begin{align}
4719: \frac{\partial \alpha(z, w)}{\partial w} = &
4720: \frac{1}{2} \sum_{i=\pm 1} iz \log_2(1+i w z)- \nonumber \\
4721: \phantom{=} & \frac{1}{2}\sum_{i=\pm 1} i\log(1+iw), \label{equ:first} \\
4722: \ln(2) \frac{\partial^2 \alpha(z, w)}{\partial w^2} = & \frac{z^2}{1-w^2 z^2}-\frac{1}{1-w^2}. \label{equ:second}
4723: \end{align}
4724: Note that equation (\ref{equ:second}) implies that $\frac{\partial^2 \alpha(z, w)}{\partial w^2}$
4725: has a positive expansion in $z$ (except for the constant term). Therefore the derivatives
4726: $\frac{\partial^{2+i} \alpha(z, w)}{\partial w^2 \partial z^i}$, $i = 1, 2$,
4727: are both positive and
4728: by symmetry of the function $\alpha(z, w)$ in its arguments $z$ and $w$ so is $\frac{\partial^{3} \alpha(z, w)}{\partial w \partial z^2}$.
4729: Finally,
4730: \begin{align*}
4731: \log(2) \frac{\partial^2 \alpha(z, w)}{\partial w \partial z} & =
4732: \frac{1}{2} \ln\frac{1+wz}{1-wz} + \frac{wz}{1-w^2 z^2} \\
4733: & = 2 w z \sum_{i \geq 0} \frac{(i+1)(w^2 z^2)^i}{2 i+1},
4734: \end{align*}
4735: which has a positive Taylor series expansion as well.
4736: This confirms our claim that the $\gexit$ kernel preserves the partial
4737: order implied by physical degradation.
4738: %
4739: %***********************************************************************
4740: %
4741: \section{Proof of Eq.~(\ref{eq:Contraction})}
4742: \label{sec:ProofContraction}
4743:
4744: In this appendix we prove the claim (\ref{eq:Contraction}).
4745: First notice that $\batta(\Tc_{\ent}(\Ldens{a}))= B_{\ent}\,
4746: \lambda(\batta(\rho(\Ldens{a})))$. Since
4747: $0\le \lambda(x)\le 1$ and $\lambda'(x)\le \lambda'(1)$, we have
4748: %
4749: \begin{align}
4750: %
4751: |\batta(T_{\ih_1}(\Ldens{a}_{1}))-&\batta(T_{\ih_2}(\Ldens{a}_{2}))|
4752: \le \label{eq:Contraction0}\\ &\lambda'(1)B_{\ih_1}\,
4753: |\batta(\rho(\Ldens{a}_{1}))-\batta(\rho(\Ldens{a}_{2}))|
4754: +|B_{\ih_1}-B_{\ih_2}|\, .\nonumber
4755: %
4756: \end{align}
4757: %
4758: In order to estimate $|\batta(\rho(\Ldens{a}_{1}))-
4759: \batta(\rho(\Ldens{a}_{2}))|$,
4760: define, for $t\in[0,1]$, $\Ldens{a}_{t} = (1-t)\Ldens{a}_1+t\Ldens{a}_2$,
4761: and write
4762: %
4763: \begin{align}
4764: %
4765: |\batta(\rho(\Ldens{a}_{1}))-\batta(\rho(\Ldens{a}_{2}))|\le
4766: \int_0^1 \left|\frac{\de \batta(\rho(\Ldens{a}_t))}{\de t}\,\right|\de t\, .
4767: \label{eq:Final1}
4768: %
4769: \end{align}
4770: %
4771: The derivative of the Battacharyya parameter
4772: is easily computed (to lighten the
4773: notation we omit hereafter the argument of $\batta(\,\cdot\,)$
4774: in the derivative).
4775: The result is most conveniently
4776: expressed in terms of densities of the variable
4777: $u\defas\sqrt{1-\tanh^2(x/2)}$, where $x$ is the log-likelihood ratio
4778: (this quantity is equivalent to the $|D|$-variable and its
4779: expectation is Battacharyya parameter).
4780: If we denote the corresponding densities by the same symbols,
4781: we get
4782: %
4783: \begin{align}
4784: %
4785: \frac{\de \batta}{\de t}
4786: =\rho'(1)
4787: \int_{0}^1\,&
4788: \sqrt{u_1^2+u_2^2- u_1^2u_2^2}\;\cdot\label{eq:DerivativeExpression}\\
4789: &\phantom{aaa}\cdot (\Ldens{a}_2(u_1)-\Ldens{a}_1(u_1))\, \Ldens{b}(u_2)\,
4790: \de u_1\,\de u_2\, ,\nonumber
4791: %
4792: \end{align}
4793: %
4794: where we introduced the density
4795: %
4796: \begin{align*}
4797: %
4798: \Ldens{b} \defas \frac{1}{\rho'(1)}\sum_{\rdegree}\rho_{\rdegree}
4799: (\rdegree-1)\,\Ldens{a}_t^{*(\rdegree-2)}\, .
4800: %
4801: \end{align*}
4802: %
4803: Using integration by parts with respect to $u_1$ in
4804: Eq.~(\ref{eq:DerivativeExpression}) and denoting by ${\mathsf A}_1$,
4805: ${\mathsf A}_2$ the distributions corresponding to densities $\Ldens{a}_1$,
4806: $\Ldens{a}_2$, we get
4807: %
4808: \begin{align*}
4809: %
4810: \frac{\de \batta}{\de t}
4811: =\rho'(1)
4812: \int_{0}^1\,&\frac{u_1(1-u_2^2)}{\sqrt{u_1^2+u_2^2-u_1^2u_2^2}}\cdot\\
4813: &\phantom{aaa}\cdot ({\mathsf A}_2(u_1)-{\mathsf A}_1(u_1))\, \Ldens{b}(u_2)\,
4814: \de u_1\,\de u_2\, ,
4815: %
4816: \end{align*}
4817: %
4818: Since $\Ldens{a}_2$ is physically degraded with respect to
4819: $\Ldens{a}_1$, ${\mathsf A}_2(u)\ge {\mathsf A}_1(u)$ for any
4820: $u\in[0,1]$. Furthermore $\int {\mathsf A}_i(v)\, \de v =
4821: \batta(\Ldens{a}_i)$. Therefore
4822: %
4823: \begin{align}
4824: %
4825: \frac{\de \batta}{\de t}
4826: =\rho'(1)
4827: [\batta(\Ldens{a}_2)-\batta(\Ldens{a}_1)]\, \Xi\, ,\label{eq:Final2}
4828: %
4829: \end{align}
4830: %
4831: where
4832: %
4833: \begin{align*}
4834: %
4835: \Xi = \int_{0}^1\,\frac{u_1(1-u_2^2)}{\sqrt{u_1^2+u_2^2-u_1^2u_2^2}}\;
4836: f(u_1)\, \Ldens{b}(u_2)\,
4837: \de u_1\,\de u_2\, ,
4838: %
4839: \end{align*}
4840: %
4841: and $f$ is a function on $[0,1]$ non negative and with unit integral.
4842: In other words, $f$ is a probability density function.
4843: Since $\sqrt{u_1^2+u_2^2-u_1^2u_2^2}\ge u_1$, we obtain the bound
4844: %
4845: \begin{align*}
4846: %
4847: \Xi &\le \int_{0}^1\,(1-u^2)\; \Ldens{b}(u)\,\de u\\
4848: &=\frac{1}{\rho'(1)}\sum_{\rdegree}\rho_{\rdegree}
4849: (\rdegree-1) \left[\int_{0}^1\,(1-u^2)\; \Ldens{a}_t(u)\right]^{\rdegree-2}\, ,
4850: %
4851: \end{align*}
4852: %
4853: where we used the definition of $\Ldens{b}$. If we further
4854: notice that $\int_{0}^1\,u\; \Ldens{a}_t(u) =\batta(\Ldens{a}_t)\ge
4855: \batta(\Ldens{a}_1)$, we get
4856: %
4857: \begin{align}
4858: %
4859: \Xi\le \frac{1}{\rho'(1)}\, \rho''(1-\batta(\Ldens{a}_1)^2)\, .
4860: \label{eq:Final3}
4861: %
4862: \end{align}
4863: %
4864: The claim follows by putting together Eqs.~(\ref{eq:Final1}),
4865: (\ref{eq:Final2}), and (\ref{eq:Final3}).
4866: %
4867: %***********************************************************************
4868: %
4869: \section{$\MAP$ Versus $\BP$ Marginals: Some Technical Details}
4870: \label{app:BPcorrect}
4871:
4872: In this appendix we present the proofs which were omitted in
4873: Sec.~\ref{sec:mapversusbp}.
4874:
4875: \begin{proof}[Lemma \ref{lem:GEXITSquareError}]
4876: Let us make a few preliminary remarks.
4877: The first one follows immediately from the definition:
4878: %
4879: \begin{align}
4880: %
4881: \E\left\{ \mu_{{\sf Y}}(Y)\mid Z=z \right\} = \mu_{{\sf Z}}(z)\, .
4882: %
4883: \end{align}
4884: %
4885: In fact, using the Markov property, the left hand side can be written as
4886: $\E\{\E[X\mid Y]\mid Z=z\}=\E\{\E[X\mid Y, Z]\mid Z=z\}$
4887: that is equal to $\E[X\mid Z=z]\equiv \mu_{{\sf Z}}(z)$.
4888:
4889: The second remark is that, by elementary calculus, for any $0\le x_0\le x\le 1$
4890: %
4891: \begin{align*}
4892: %
4893: k(x)\le k(x_0)-\frac{1}{2}\, K\, (x^2-x_0^2)\, .
4894: %
4895: \end{align*}
4896:
4897: Finally, for any random variable $W$, taking values in $[0,1]$, we have
4898: (here ${\rm Var}(W)$ is the variance of $W$):
4899: %
4900: \begin{align*}
4901: %
4902: \E\,k(W)\le k(\E\, W)-\frac{1}{2}\, K\, {\rm Var}(W)\, .
4903: %
4904: \end{align*}
4905: %
4906: In fact, by Taylor expansion $k(W)\le k(w_0)+k'(w_0)(W-w_0)
4907: -\frac{1}{2}\, K\, (W-w_0)$, for any $w_0\in[0,1]$. The claim is proved by
4908: taking expectation of both sides and setting $w_0= \E W$.
4909:
4910: These ingredients are put together as follows
4911: (here we use the shorthands $\mu_{{\sf Y}}$ and $\mu_{{\sf Z}}$
4912: for, respectively, $\mu_{{\sf Y}}(Y)$ and $\mu_{{\sf Z}}(Z)$)
4913: %
4914: \begin{eqnarray*}
4915: %
4916: &&\hspace{-0.5cm}\E[k(|\mu_{{\sf Y}}|)] = \E\left\{\E[k(|\mu_{{\sf Y}}|)|Z] \right\}\\
4917: &&\le \E\left\{k(\E[|\mu_{{\sf Y}}||Z]) -\frac{1}{2}\, K\,
4918: {\rm Var}(|\mu_{{\sf Y}}|\, \mid Z)\right\}\\
4919: &&\le \E\left\{k(|\E[\mu_{{\sf Y}}|Z]|)-\frac{1}{2}\, K\,
4920: \left(\E[|\mu_{{\sf Y}}||Z]^2-\E[\mu_{{\sf Y}}|Z]^2\right)-\right.\\
4921: &&\left.\hspace{5.cm}
4922: -\frac{1}{2}\, K\,{\rm Var}(|\mu_{{\sf Y}}|\, \mid Z)\right\}\\
4923: && = \E\left\{k(|\E[\mu_{{\sf Y}}|Z]|)-\frac{1}{2}\, K\,
4924: \E[(\mu_{{\sf Y}}-\E[\mu_{{\sf Y}}|Z])^2|Z]\right\}\\
4925: && = E\left\{k(|\mu_{{\sf Z}}|)-\frac{1}{2}\, K\,
4926: \E[(\mu_{{\sf Y}}-\mu_{{\sf Z}})^2|Z]\right\}\\
4927: && = E[k(|\mu_{{\sf Z}}|)]-\frac{1}{2}\, K\,
4928: \E[(\mu_{{\sf Y}}-\mu_{{\sf Z}})^2]\, ,
4929: %
4930: \end{eqnarray*}
4931: %
4932: which completes the proof.
4933: %
4934: \end{proof}
4935: %
4936:
4937: %
4938: \begin{proof}[Lemma \ref{lem:ExtrinsicDistortion}]
4939: %
4940: We claim (and will prove later) that
4941: %
4942: \begin{eqnarray*}
4943: %
4944: \left|\tilde{\mu}^{\BPsmall,\ell}_i(Y)-\tilde{\mu}_i(Y)
4945: \right|\le e^{|l(Y_i)|}\left|\mu^{\BPsmall,\ell}_i(Y)-\mu_i(Y)\right|\, ,
4946: %
4947: \end{eqnarray*}
4948: %
4949: where $l(y_i)$ is the log-likelihood associated to the channel output
4950: $y_i$.
4951: If we square and take expectation with respect to $Y$
4952: (recalling that $\mu^{\BPsmall,\ell}_i(Y)$, $\mu_i(Y)$ do not depend upon
4953: $Y_i$), we get
4954: %
4955: \begin{eqnarray*}
4956: %
4957: \E\left\{\left|\tilde{\mu}^{\BPsmall,\ell}_i(Y)-\tilde{\mu}_i(Y)
4958: \right|^2\right\}\le C\,
4959: \E\left\{\left|\mu^{\BPsmall,\ell}_i(Y)-\mu_i(Y)\right|^2\right\}\, .
4960: %
4961: \end{eqnarray*}
4962: %
4963: The thesis follows by summing over $i$.
4964:
4965: We are left with the task of proving the first claim above.
4966: We recall that the conditional expectations can be represented in terms
4967: of extrinsic log-likelihoods as
4968: %
4969: \begin{eqnarray*}
4970: %
4971: \mu_i(y) &= &\tanh\left[\frac{1}{2}\phi^{}_i(y_{\sim i})\right]
4972: \, ,\\
4973: \tilde{\mu}_i(y) &=& \tanh\left[\frac{1}{2}(l(y_i)+\phi^{}_i(y_{\sim i}))\right]\, .
4974: %
4975: \end{eqnarray*}
4976: %
4977: Analogous formulae hold if we replace $\mu_i(y)$ (respectively
4978: $\tilde{\mu}_i(y)$) with $\mu^{\BPsmall,\ell}_i(y)$ (respectively
4979: $\tilde{\mu}^{\BPsmall,\ell}_i(y)$) and
4980: $\phi^{}_i(y_{\sim i})$ with $\phi^{\BPsmall,\ell}_i(y_{\sim i})$.
4981: The claim follows immediately form the following
4982: calculus exercise below.
4983: \end{proof}
4984:
4985: %
4986: \begin{fact}
4987: %
4988: For any $x_1,x_2,z\in{\mathbb R}$
4989: %
4990: \begin{align*}
4991: %
4992: |\tanh(x_1+z)-\tanh(x_2+z)&|\le \\
4993: &e^{2|z|} |\tanh(x_1)-\tanh(x_2)| \, .
4994: %
4995: \end{align*}
4996: %
4997: \begin{proof}
4998: %
4999: Consider, without loss of generality, $x_1>x_2$ and $z<0$.
5000: It is simple to realize that, for any $x\in{\mathbb R}$
5001: %
5002: \begin{eqnarray*}
5003: %
5004: 1+\tanh(x+z)&\le & 1+\tanh(x)\, ,\\
5005: 1-\tanh(x+z)&\le & e^{-2z}(1-\tanh(x))\, .
5006: %
5007: \end{eqnarray*}
5008: %
5009: The last statement follows by writing
5010: $1+e^{2(x+z)}\ge e^{2z}(1+e^{2x})$ and taking the inverse of both sides.
5011:
5012: The thesis is proved by multiplying these inequalities, and
5013: integrating over $x\in[x_1,x_2]$.
5014: %
5015: \end{proof}
5016: %
5017: \end{fact}
5018: %
5019: %***********************************************************************
5020: %
5021:
5022: % bibliography
5023: % ------------
5024: \bibliographystyle{IEEEtran} %plain
5025:
5026: \newcommand{\SortNoop}[1]{}
5027: \begin{thebibliography}{10}
5028: \providecommand{\url}[1]{#1}
5029: \csname url@rmstyle\endcsname
5030: \providecommand{\newblock}{\relax}
5031: \providecommand{\bibinfo}[2]{#2}
5032: \providecommand\BIBentrySTDinterwordspacing{\spaceskip=0pt\relax}
5033: \providecommand\BIBentryALTinterwordstretchfactor{4}
5034: \providecommand\BIBentryALTinterwordspacing{\spaceskip=\fontdimen2\font plus
5035: \BIBentryALTinterwordstretchfactor\fontdimen3\font minus
5036: \fontdimen4\font\relax}
5037: \providecommand\BIBforeignlanguage[2]{{%
5038: \expandafter\ifx\csname l@#1\endcsname\relax
5039: \typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
5040: \typeout{** loaded for the language `#1'. Using the pattern for}%
5041: \typeout{** the default language instead.}%
5042: \else
5043: \language=\csname l@#1\endcsname
5044: \fi
5045: #2}}
5046:
5047: \bibitem{MMRU04}
5048: C.~M{\'e}asson, A.~Montanari, T.~Richardson, and R.~Urbanke, ``Life above
5049: threshold: From list decoding to area theorem and {MSE},'' in \emph{Proc. of
5050: the IEEE Inform. Theory Workshop}, San Antonio, Texas, October 24--29 2004.
5051:
5052: \bibitem{MMRU05}
5053: ------, ``Maximum a posteriori decoding and turbo codes for general memoryless
5054: channels,'' in \emph{Proc. of the IEEE Int. Symposium on Inform. Theory},
5055: Adelaide, Sept. 2005.
5056:
5057: \bibitem{MeU03}
5058: C.~M{\'e}asson and R.~Urbanke, ``An upper-bound on the {ML} thresholds of
5059: {LDPC} ensembles over the {BEC},'' in \emph{Proc. 41th Annual Allerton
5060: Conference on Communication, Control and Computing}, Monticello, IL, October
5061: 2003.
5062:
5063: \bibitem{MMU04}
5064: C.~M{\'e}asson, A.~Montanari, and R.~Urbanke, ``Maxwell's construction: The
5065: hidden bridge between maximum-likelihood and iterative decoding,'' in
5066: \emph{Proc. of the IEEE Int. Symposium on Inform. Theory}, Chicago, 2004.
5067:
5068: \bibitem{MMU05}
5069: ------, ``Maxwell's construction: The hidden bridge between iterative and
5070: maximum a posteriori decoding,'' 2005, submitted to IEEE Transactions on
5071: Information Theory.
5072:
5073: \bibitem{GSV04c}
5074: D.~Guo, S.~Shamai, and S.~Verdu, ``Mutual information and conditional mean
5075: estimation in poisson channels,'' in \emph{Proc. of the IEEE Inform. Theory
5076: Workshop}, San Antonio, Texas, October 24--29 2004.
5077:
5078: \bibitem{GSV05}
5079: ------, ``Mutual information and minimum mean-square error in gaussian
5080: channels,'' \emph{IEEE Trans. Inform. Theory}, vol.~51, pp. 1261--1882, Apr.
5081: 2005.
5082:
5083: \bibitem{Mac05}
5084: N.~Macris, ``Correlation inequalities: a useful tool in the theory of {LDPC}
5085: codes,'' in \emph{Proc. of the IEEE Int. Symposium on Inform. Theory},
5086: Adelaide, Australia, Sept. 2005.
5087:
5088: \bibitem{GSV05i}
5089: D.~Guo, S.~Shamai, and S.~Verdu, ``Additive non-gaussian noise channels: Mutual
5090: information and conditional mean estimation,'' in \emph{Proc. of the IEEE
5091: Int. Symposium on Inform. Theory}, Adelaide, Australia, Sept. 2005.
5092:
5093: \bibitem{Zak05}
5094: M.~Zakai, ``On mutual information, likelihood-ratios and estimation error for
5095: the additive gaussian channel,'' 2005, submitted IEEE IT.
5096:
5097: \bibitem{RiU05}
5098: T.~Richardson and R.~Urbanke, \emph{Modern Coding Theory}.\hskip 1em plus 0.5em
5099: minus 0.4em\relax Cambridge University Press, 2005, in preparation.
5100:
5101: \bibitem{LMSS01}
5102: M.~Luby, M.~Mitzenmacher, A.~Shokrollahi, and D.~A. Spielman, ``Efficient
5103: erasure correcting codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~47,
5104: no.~2, pp. 569--584, Feb. 2001.
5105:
5106: \bibitem{LMSS01b}
5107: ------, ``Improved low-density parity-check codes using irregular graphs,''
5108: \emph{IEEE Trans. Inform. Theory}, vol.~47, pp. 585--598, 2001.
5109:
5110: \bibitem{RiU01}
5111: T.~Richardson and R.~Urbanke, ``The capacity of low-density parity check codes
5112: under message-passing decoding,'' \emph{IEEE Trans. Inform. Theory}, vol.~47,
5113: pp. 599--618, Feb. 2001.
5114:
5115: \bibitem{RSU01}
5116: T.~Richardson, A.~Shokrollahi, and R.~Urbanke, ``Design of capacity-approaching
5117: irregular low-density parity-check codes,'' \emph{IEEE Trans. Inform.
5118: Theory}, vol.~47, pp. 619--637, Feb. 2001.
5119:
5120: \bibitem{AKtB04}
5121: A.~Ashikhmin, G.~Kramer, and S.~ten Brink, ``Extrinsic information transfer
5122: functions: model and erasure channel property,'' \emph{IEEE Trans. Inform.
5123: Theory}, vol.~50, no.~11, pp. 2657--2673, Nov. 2004.
5124:
5125: \bibitem{teB99a}
5126: S.~ten Brink, ``Convergence of iterative decoding,'' \emph{Electron. Lett.},
5127: vol.~35, no.~10, pp. 806--808, May 1999.
5128:
5129: \bibitem{teB99b}
5130: ------, ``Iterative decoding for multicode {CDMA},'' in \emph{Proc. IEEE VTC},
5131: vol.~3, May 1999, pp. 1876--1880.
5132:
5133: \bibitem{tBr00}
5134: S.~{ten Brink}, ``Designing iterative decoding schemes with the extrinsic
5135: information transfer chart,'' \emph{AEU Int. J. Electron. Commun.}, vol.~54,
5136: pp. 389--398, 2000.
5137:
5138: \bibitem{teB00}
5139: S.~ten Brink, ``Iterative decoding trajectories of parallel concatenated
5140: codes,'' in \emph{Proc. 3rd IEEE/ITG Conf. Source Channel Coding}, Jan. 2000,
5141: pp. 75--80.
5142:
5143: \bibitem{teB01}
5144: ------, ``Convergence behavior of iteratively decoded parallel concatenated
5145: codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~49, no.~10, pp. 1727--1737,
5146: Oct. 2001.
5147:
5148: \bibitem{CoT91}
5149: T.~M. Cover and J.~A. Thomas, \emph{Elements of Information Theory}.\hskip 1em
5150: plus 0.5em minus 0.4em\relax New York: Wiley, 1991.
5151:
5152: \bibitem{LHHH03}
5153: I.~Land, P.~Hoeher, S.~Huettinger, and J.~B. Huber, ``Bounds on information
5154: combining,'' in \emph{Proc. of the 3rd Int. Symp. on Turbo Codes and Related
5155: Topics}, Brest, France, Sept. 2003.
5156:
5157: \bibitem{HuH03}
5158: S.~Huettinger and J.~B. Huber, ``Information processing and combining in
5159: channel coding,'' in \emph{Proc. of the 3rd Int. Symp. on Turbo Codes and
5160: Related Topics}, Brest, France, Sept. 2003.
5161:
5162: \bibitem{SSZ03}
5163: I.~Sutskover, S.~Shamai, and J.~Ziv, ``Extremes of information combining,'' in
5164: \emph{Proc. 41th Annual Allerton Conference on Communication, Control and
5165: Computing}, Monticello, IL, 2003.
5166:
5167: \bibitem{GSV04}
5168: D.~Guo, S.~Shamai, and S.~Verd{\'u}, ``Mutual information and {MMSE} in
5169: gaussian channels,'' in \emph{IEEE International Symposium on Information
5170: Theory}, Chicago, USA, June 27 - July 2 2004, p. 349.
5171:
5172: \bibitem{andrea}
5173: A.~Montanari, ``The glassy phase of {G}allager codes,'' \emph{Eur. Phys. J. B},
5174: vol.~23, pp. 121--136, 2001.
5175:
5176: \bibitem{Mon04}
5177: ------, ``Tight bounds for {LDPC} and {LDGM} codes under {MAP} decoding,''
5178: \emph{IEEE Trans. Inform. Theory}, vol.~51, no.~9, pp. 3221 -- 3246, Sept.
5179: 2005.
5180:
5181: \bibitem{WiS05}
5182: G.~Wiechman and I.~Sason, ``Improved bounds on the parity-check density and
5183: achievable rates of binary linear block codes with applications to {LDPC}
5184: codes,'' May 2005, submitted to {IEEE IT}, arXiv:cond-math/cond-mat/0505057.
5185:
5186: \bibitem{BKLM02}
5187: D.~Burshtein, M.~Krivelevich, S.~L. Litsyn, and G.~Miller, ``Upper bounds on
5188: the rate of {LDPC} codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~48,
5189: no.~9, pp. 2437--2449, Sept. 2002.
5190:
5191: \bibitem{Shir96}
5192: A.~N. Shiryaev, \emph{Probability}.\hskip 1em plus 0.5em minus 0.4em\relax
5193: Springer, 1996.
5194:
5195: \bibitem{Bro93}
5196: R.~F. Brown, \emph{A topological introduction to nonlinear analysis}.\hskip 1em
5197: plus 0.5em minus 0.4em\relax Birkh{\"a}user, 1993.
5198:
5199: \bibitem{LHHH05}
5200: I.~Land, P.~Hoeher, S.~Huettinger, and J.~B. Huber, ``Bounds on information
5201: combining,'' \emph{IEEE Trans. Inform. Theory}, vol.~51, no.~2, pp. 612--619,
5202: 2005.
5203:
5204: \bibitem{SSZ05}
5205: I.~Sutskover, S.~Shamai, and J.~Ziv, ``Constrained information combining:
5206: Theory and applications for {LDPC} coded systems,'' 2005, submitted IEEE IT.
5207:
5208: \bibitem{BaN04}
5209: K.~Bhattad and K.~R. Narayanan, ``An {MSE} based transfer chart to analyze
5210: iterative decoding schemes,'' in \emph{Proc. of the Allerton Conf. on
5211: Commun., Control and Computing}, Monticello, IL, USA, Oct. 2004.
5212:
5213: \bibitem{Sho00}
5214: A.~Shokrollahi, ``Capacity-achieving sequences,'' in \emph{Codes, Systems, and
5215: Graphical Models}, ser. IMA Volumes in Mathematics and its Applications,
5216: B.~Marcus and J.~Rosenthal, Eds., vol. 123.\hskip 1em plus 0.5em minus
5217: 0.4em\relax Springer-Verlag, 2000, pp. 153--166.
5218:
5219: \bibitem{OsS01}
5220: P.~Oswald and A.~Shokrollahi, ``Capacity achieving sequences for the erasure
5221: channel,'' in \emph{Proceedings of the International Symposium on Information
5222: Theory, Washington DC}, 2001, p.~48.
5223:
5224: \bibitem{AKtB02a}
5225: A.~Ashikhmin, G.~Kramer, and S.~ten Brink, ``Extrinsic information transfer
5226: functions: a model and two properties,'' in \emph{Proc. of Conference on
5227: Information Sciences and Systems (CISS)}, Princeton University, Mar. 2002.
5228:
5229: \bibitem{AKTB02}
5230: A.~Ashikhmin, G.~Kramer, and S.~{ten Brink}, ``Code rate and the area under
5231: extrinsic information transfer curves,'' in \emph{Proc. of the IEEE Int.
5232: Symposium on Inform. Theory}, Lausanne, Switzerland, June 30--July 5 2002, p.
5233: 115.
5234:
5235: \bibitem{MeU02jccc}
5236: C.~M{\'e}asson and R.~Urbanke, ``Asymptotic analysis of turbo codes over the
5237: binary erasure channel,'' in \emph{Proc. of the 12th Joint Conference on
5238: Communications and Coding}, Saas Fee, Switzerland, March 2002.
5239:
5240: \bibitem{MeU03a}
5241: ------, ``Further analytic properties of {EXIT}-like curves and applications,''
5242: in \emph{Proc. of the IEEE Int. Symposium on Inform. Theory}, Yokohama,
5243: Japan, June 29--July 4 2003, p. 266.
5244:
5245: \bibitem{For05}
5246: G.~D. Forney, ``Lecture notes,'' 2005, mIT.
5247:
5248: \bibitem{HOP96}
5249: J.~Hagenauer, E.~Offer, and L.~Papke, ``Iterative decoding of binary block and
5250: convolutional codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~42, pp.
5251: 429--445, Mar. 1996.
5252:
5253: \bibitem{Zem65}
5254: A.~H. Zemanian, \emph{Distribution Theory and Transform Analysis}.\hskip 1em
5255: plus 0.5em minus 0.4em\relax New York, USA: Dover Publications, 1965.
5256:
5257: \bibitem{Guo04}
5258: D.~Guo, ``Gaussian channels: Information, estimation and multiuser detection,''
5259: Ph.D. dissertation, Princeton University, 2004.
5260:
5261: \end{thebibliography}
5262:
5263:
5264: %\begin{biography}{Michael Shell}
5265: %Biography text here.
5266: %\end{biography}
5267:
5268: % bibliography
5269: % ------------
5270: %\bibliographystyle{IEEEtran} %plain
5271: %\bibliography{lth,lthpub}
5272:
5273: % if you will not have a photo at all:
5274: %\begin{biographynophoto}{John Doe}
5275: %Biography text here.
5276: %\end{biographynophoto}
5277:
5278: % insert where needed to balance the two columns on the last page
5279: %\newpage
5280:
5281: %\begin{biographynophoto}{Jane Doe}
5282: %Biography text here.
5283: %\end{biographynophoto}
5284:
5285: % You can push biographies down or up by placing
5286: % a \vfill before or after them. The appropriate
5287: % use of \vfill depends on what kind of text is
5288: % on the last page and whether or not the columns
5289: % are being equalized.
5290:
5291: %\vfill
5292:
5293: % Can be used to pull up biographies so that the bottom of the last one
5294: % is flush with the other column.
5295: %\enlargethispage{-5in}
5296:
5297: % that's all folks
5298: \end{document}
5299:
5300:
5301: