1: % The following commands allow CVS to insert the
2: % current information regarding this file:
3: % ----------------------------------------------
4: % $RCSfile: main.tex,v $
5: % $Date: 2005/06/17 14:17:27 $
6: % $Author: cyril $
7: % $Revision: 1.441 $
8: % $Source: /cvs/lthc/projects/maxwell/becpaper/main.tex,v $
9: % $State: Exp $
10: % -----------------------------------------------
11: %\documentclass[peerreview]{IEEEtran}
12: \documentclass[journal]{IEEEtran}
13: %\documentclass[onecolumn,12pt,draft]{IEEEtran}
14:
15: % package list
16: \usepackage{amssymb,amsmath,amsfonts,rotating}%amsfonts,amsthm
17: \usepackage[noadjust]{cite}
18: \usepackage{color} %temporary packages
19: %%\usepackage[vflt]{floatflt}
20: \usepackage{floatfig}
21:
22: %\usepackage{stfloats}
23: % definition list
24: %\input{definition} % macros
25:
26: %---------------------------------------------------------------------------------
27: % Usefull macro:
28: % sept.2004. c.m.
29: %---------------------------------------------------------------------------------
30:
31:
32: %---------------------------------------------------------------------------------
33: % temporary options:
34: % comments and corrections when writing
35: %---------------------------------------------------------------------------------
36: \definecolor{TODO}{rgb}{0.6,0.6,0.6} % TO DO!!!!
37: \newcommand{\todo}[1]{{\color{TODO}TO DO: {#1}}\marginpar{\color{TODO}$\leftarrow$ TO DO!}}
38: \definecolor{TOCHECK}{rgb}{0.8,0.8,0.8} % TO CKECK!!!!
39: \newcommand{\tocheck}[1]{\marginpar{\color{TOCHECK}$\leftarrow$ TO CHECK: {#1}}}
40: %---------------------------------------------------------------------------------
41:
42:
43: %---------------------------------------------------------------------------------
44: % LaTex environment
45: %---------------------------------------------------------------------------------
46: \newtheorem{theorem}{Theorem}
47: \newcommand{\btheo}{\begin{theorem}}
48: \newcommand{\etheo}{\end{theorem}}
49: \newcommand{\bproof}{\begin{proof}}
50: \newcommand{\eproof}{\end{proof}}
51: \newtheorem{definition}{Definition}
52: \newcommand{\bdefi}{\begin{definition}}
53: \newcommand{\edefi}{\end{definition}}
54: \newtheorem{fact}{Fact}
55: \newcommand{\bprop}{\begin{fact}}
56: \newcommand{\eprop}{\end{fact}}
57: \newtheorem{corollary}{Corollary}
58: \newcommand{\bcor}{\begin{corollary}}
59: \newcommand{\ecor}{\end{corollary}}
60: \newtheorem{example}{Example}
61: \newcommand{\bex}{\begin{example}}
62: \newcommand{\eex}{\end{example}}
63: \newtheorem{lemma}{Lemma}
64: \newcommand{\blemma}{\begin{lemma}}
65: \newcommand{\elemma}{\end{lemma}}
66: \newtheorem{remark}{Remark}
67: \newcommand{\bremark}{\begin{remark}}
68: \newcommand{\eremark}{\end{remark}}
69: \newtheorem{conjecture}{Conjecture}
70: \newcommand{\bconj}{\begin{conjecture}}
71: \newcommand{\econj}{\end{conjecture}}
72: %---------------------------------------------------------------------------------
73:
74:
75: %---------------------------------------------------------------------------------
76: % Mathematics:
77: %---------------------------------------------------------------------------------
78: %---------> international notations
79: \newcommand{\reals}{\ensuremath{\mathbb{R}}}
80: \newcommand{\naturals}{\ensuremath{\mathbb{N}}}
81: %---------> mynotations
82: \newcommand{\expectation}{\ensuremath{\mathbb{E}}}
83: \newcommand{\argmin}{\ensuremath{\text{argmin}}}
84: \newcommand{\defas}{\ensuremath{\stackrel{{\vartriangle}}{=}}}
85: %{\ensuremath{\stackrel{\triangle}{=}}}%{\ensuremath{:=}}%{\colon=}} %\stackrel{\vartriangle}{=}
86: %---------------------------------------------------------------------------------
87:
88:
89: %---------------------------------------------------------------------------------
90: % Coding and Information Theory:
91: %---------------------------------------------------------------------------------
92: %---------> Modulation (transmitted bit)
93: \def\0{{\tt 0}} % Ex.: BPSK modulation => 0 is encoded into +1
94: \def\1{{\tt 1}} % Ex.: BPSK modulation => 1 is encoded into -1
95: \def\?{{\tt *}} % erasure symbol
96: \def\g{{\tt g}} % guess symbol
97:
98: \def\edge{{\tt e}} % edge
99: \newcommand{\gp}{\ensuremath{\gamma}} % proba for guessing a bit
100: \newcommand{\chm}[1]{\ensuremath{{{\mu^{{\tt{\epsilon}}}_{#1}}}}} %
101: %\def\chm{{\tt {z} }} % channel message
102: \newcommand{\lrm}[1]{\ensuremath{{{\mu^{{\tt{x}}}_{#1}}}}} % left-to-right message
103: \newcommand{\rlm}[1]{\ensuremath{{{\mu^{{\tt{y}}}_{#1}}}}} % right-to-left message
104: \newcommand{\m}{\ensuremath{{{\mu}}}} % message
105: \def\lra{{\tt {x}}} % left-to-right density
106: \def\rla{{\tt {y}}} % right-to-left density
107: %---------> LDPC and Turbo notations
108: \newcommand{\graph}{{\ensuremath{\tt G}}}
109: %\newcommand{\code}{{\ensuremath{\tt C}}}
110: \newcommand{\ldpc}{{{\ensuremath{\text{LDPC}}}}}
111: \newcommand{\ddp}{dd~pair~} %degree distribution pair %d.d.
112: \newcommand{\ddps}{dd~pairs~} %degree distribution pair %d.d.
113: \newcommand{\maxwell}{M~} %Maxwell
114: %\DeclareMathOperator{\ldpc}{LDPC}
115: %\DeclareMathOperator{\graph}{G}
116: \newcommand{\n}{\ensuremath{n}} % code length
117: \newcommand{\drate}{r} % design rate
118: \newcommand{\rate}{\ensuremath{r}} % design rate
119: \newcommand{\ledge}{\ensuremath{\lambda}} % ledge (edge perspective)
120: \newcommand{\redge}{\ensuremath{\rho}} % redge (edge perspective
121: \newcommand{\lnode}{\ensuremath{\Lambda}} % lnode (node perspective)
122: \newcommand{\rnode}{\ensuremath{\Gamma}} % rnode (node perspective)
123: \newcommand{\ldegree}{{\ensuremath{\ensuremath{\tt l}}}} % left degree (regular codes)
124: \newcommand{\rdegree}{{\ensuremath{\ensuremath{\tt r}}}} % right degree (regular codes)
125: %---------> EXIT quantities, thresholds and channel
126: \newcommand{\ih}{\ensuremath{\epsilon}} % intrinsic (or input or channel) entropy
127: \newcommand{\cp}{\epsilon} % channel parameter
128: \newcommand{\xh}{\ensuremath{h}} % extrinsic (or EXIT) entropy
129: \newcommand{\ML}{\ensuremath{\text{\tiny ML}}} % Maximum Likelihood
130: \newcommand{\MAP}{\ensuremath{\text{\tiny MAP}}} % Maximum A Posteriori
131: \newcommand{\BP}{\ensuremath{\text{\tiny BP}}} % Belief Propagation
132: \newcommand{\EBP}{\ensuremath{\text{\tiny EBP}}} % Extended Belief Propagation
133: \newcommand{\Sh}{\ensuremath{\text{\tiny Sh}}} % Shannon
134: \newcommand{\SC}{\ensuremath{\text{\tiny SC}}} % Stability Condition
135:
136: %\newcommand{\iteration}{\ensuremath{\ell}} % iteration index during BP iterative decoding
137:
138: \newcommand{\xl}{\ensuremath{{\tt{x}}}}%\xi}} %decoding parameter, FIND A LETTER FOR IT!!!
139: % = left EXIT entropy
140: \newcommand{\xr}{\ensuremath{{\tt{y}}}}%\xi}} % rexit or right EXIT entropy
141: \newcommand{\lxl}{\ensuremath{\underline{\xl}}} %lower parameter
142: \newcommand{\uxl}{\ensuremath{\overline{\xl}}} %upper parameter
143: %--- non-math-mode macros
144: \newcommand{\exitentropy}{EXIT~} %EXIT entropy
145:
146: \newcommand{\qed}{{\hfill \footnotesize $\blacksquare$}}
147: \renewcommand{\mid}{\,|\,}
148: \newcommand{\tablespace}{\vspace{10pt}}
149:
150:
151: \newcommand{\hb}{\bar{h}}
152: \newcommand{\zh}{\hat{z}}
153: \newcommand{\de}{\text{d}}
154: \newcommand{\cX}{{\cal X}}
155: \newcommand{\nddp}{\Xi}
156: \newcommand{\zl}{{\tt z}}
157: \newcommand{\cN}{{\cal N}}
158: \newcommand{\pr}{{\rm Pr}}
159: \newcommand{\List}{\Theta}
160: \newcommand{\ind}{{\mathbb I}}
161: \newcommand{\cW}{{\cal W}}
162:
163: \newcommand{\entropy}{\hat{H}} % logarithm of the number of running copies
164: \newcommand{\iter}{t}
165: \newcommand{\drmax}{{\tt r}_{\text{max}}}
166:
167:
168: %\hyphenation{op-tical net-works semi-conduc-tor}
169:
170: \begin{document}
171: \initfloatingfigs
172: {
173: \title{Maxwell Construction: The Hidden Bridge between
174: Iterative and Maximum a Posteriori Decoding} %
175: \author{
176: Cyril~M\'easson$^\dagger$ %
177: \thanks{$\dagger$ EPFL, School for Computer and Communication Sciences, CH-1015 Lausanne, Switzerland. E-mail: cyril.measson@epfl.ch}, %
178: Andrea~Montanari$^*$ %
179: \thanks{$*$ ENS, Laboratoire de Physique Th\'eorique, F-75231 Paris, France. E-mail: montanar@lpt.ens.fr} %
180: and R\"udiger Urbanke %
181: $^\ddagger$\thanks{$\ddagger$ EPFL, School for Computer and Communication Sciences, CH-1015 Lausanne, Switzerland. E-mail: ruediger.urbanke@epfl.ch} %
182: \thanks{Parts of the material were presented in \cite{MeU03b,Mon01c,MMU04, MMRU04}.}
183: } %
184: %\markboth{Draft MMU -- \today}
185: \maketitle
186: }
187:
188: \begin{abstract}
189: There is a fundamental relationship between belief propagation
190: and maximum a posteriori decoding.
191: A decoding algorithm, which we call the Maxwell decoder, is introduced and
192: provides a constructive description of this relationship.
193: Both, the algorithm itself and the analysis of the new decoder are reminiscent
194: of the Maxwell construction in thermodynamics.
195: This paper investigates in detail the case of transmission
196: over the binary erasure channel, while the extension to general binary
197: memoryless channels is discussed in a companion paper.
198: \end{abstract}
199:
200: \begin{keywords}
201: belief propagation, maximum a posteriori, maximum likelihood, Maxwell construction,
202: threshold, phase transition, Area Theorem, \exitentropy curve, entropy
203: %IEEEtran, journal, \LaTeX, paper, template.
204: \end{keywords}
205:
206: %\IEEEpeerreviewmaketitle
207:
208: \section{Introduction}
209: %\subsection{Context}
210: \PARstart{I}{t}
211: %It
212: is a key result,
213: and the starting point of iterative coding, that belief
214: propagation (BP) is optimal on trees. See, e.g., \cite{Per88,Wib96,AjM00,KFL01}.
215: However, trees with bounded state size appear not to be powerful enough
216: models to allow transmission arbitrarily close to capacity. For instance, it
217: is known that in the setting of standard binary Tanner graphs the error
218: probability of codes
219: defined on trees is lower bounded by a constant
220: which only depends on the channel and the rate of the code \cite{ETV99,RiU05}.
221: The general wisdom
222: is therefore to apply BP decoding to graphs with loops and to consider this
223: type of decoding as a (typically) strictly suboptimal attempt to perform maximum
224: a posteriori (MAP) bit decoding. One would therefore not expect any link
225: between the BP and the MAP decoder except for the obvious suboptimality of the
226: BP decoder.
227:
228: This contribution demonstrates that there is a fundamental relationship between
229: BP and MAP decoding which appears in the limit of large blocklengths.
230: This relationship is furnished by the so-called Maxwell (M) decoder.
231: The \maxwell decoder combines the BP decoder with a ``guessing'' device to
232: perform MAP decoding. It is possible to analyze the performance of the \maxwell decoder
233: in terms of the \exitentropy curve introduced in \cite{teB01}.
234: This analysis leads to a precise characterization of
235: how difficult it is to convert the BP decoder into a MAP decoder and
236: this ``gap'' between the MAP and BP decoder has a pleasing
237: graphical interpretation in terms of an
238: area under the \exitentropy curve.\footnote{The \exitentropy curve is here the \exitentropy curve associated to the
239: iterative coding system and not to its individual component codes. This differs from
240: the original \exitentropy chart context presented in \cite{teB01}.} Further, the MAP threshold is determined by
241: a balance between two areas representing the number of guesses and the reduction
242: in uncertainty, respectively.
243: The analysis gives also rise to a generalized Area Theorem, see also
244: \cite{AKtB02}, and it provides an alternative tool for proving area-like
245: results.
246:
247: The concept of a ``BP decoder with guesses'' itself is not new. In \cite{PiF04}
248: the authors introduced such a decoder in order to improve the performance
249: of the BP decoder. Our motivation though is quite different.
250: Whereas, from a practical point of view, such enhancements work best
251: for relatively small code lengths, or to clean up error floors, we are interested
252: in the asymptotic setting in which the unexpected
253: relationship between the MAP decoder and the BP decoder emerges.
254: %
255: %\subsection{Setup}
256: \subsection{Preliminaries}
257: %
258: Assume that transmission takes place over a binary erasure channel with
259: parameter $\ih$, call it BEC($\ih$). More precisely, the transmitted bit $x_i$ at time $i$, $x_i \in{\cal{X}}\defas\{\0,\1\}$, is erased with probability $\ih$.
260: The channel output is the random variable $Y_i$ which takes values in
261: ${\cal{Y}}\defas\{\0,\?,\1\}$. To be concrete, we will exemplify all
262: statements using Low-Density Parity-Check (LDPC) code ensembles \cite{Gal63}.
263: However, the results extend to other ensembles like, e.g., Generalized
264: $\ldpc$ or turbo codes, and we will state the results in a general form.
265: %
266: For an in-depth introduction to the analysis of $\ldpc$ ensembles see, e.g.,
267: \cite{LMSS01,LMSS01b,RiU01,RSU01}. For convenience of the reader, and to settle
268: notation, let us briefly review some key statements. The degree
269: distribution (dd) pair
270: $\left(\ledge(\xl),\redge(\xl)\right)=(\sum_j \ledge_j \xl^{j-1},\sum_j \redge_j \xl^{j-1})$
271: represents the degree distribution of the graph from the {\em edge} perspective.
272: We consider the ensemble $\ldpc(\ledge,\redge,\n)$
273: of such graphs of length $\n$ and we are interested in its
274: asymptotic average performance
275: (when the blocklength $\n\to \infty$).
276: %
277: This ensemble can equivalently be described by
278: $\nddp \defas (\lnode(\xl),\rnode(\xl)) =
279: (\sum_j \lnode_j \xl^{j}, \sum_j \rnode_j \xl^{j})$,
280: which is the \ddp from the {\em node} perspective\footnote{The changes of
281: representation are obtained via $\lnode(\xl)=(1/\int\ledge){\int_0^\xl \ledge(u)\text{d}u}$,
282: $\rnode(\xl)=(1/\int\redge){\int_0^\xl \redge(u)\text{d}u}$, $\ledge(\xl) = \lnode'(\xl)/\lnode'(1)$
283: and
284: $\redge(\xl) = \rnode'(\xl)/\rnode'(1)$.}.
285: %
286: An important characteristic of the
287: ensemble $\ldpc(\ledge,\redge,\n)$ is the {\em design rate}
288: $\drate \defas 1-\int\redge/\int\ledge = 1-\lnode'(1)/\Gamma'(1)$. We will
289: write $\drate=\drate(\ledge,\redge)$ or $\drate = \drate(\lnode,\rnode)$
290: whenever we regard the design rate as a function of the degree distribution pair.
291:
292: The BP threshold, call it $\ih^\BP=\ih^\BP(\ledge,\redge)$, is defined in \cite{LMSS01,LMSS01b,RiU01,RSU01} as
293: $
294: \ih^\BP \defas \text{sup} \{ \ih\in[0,1]: \ih \ledge(1-\redge(1-\xl))<\xl, \forall \xl\in(0,1]\}.
295: $
296: Operationally, if we transmit at $\ih<\ih^\BP$ and use a BP decoder, then all
297: bits except possibly a sub-linear fraction can be recovered when $\n\to\infty$.
298: On the other hand, if $\ih\geq\ih^\BP$, then a fixed fraction of bits remains erased
299: after BP decoding when $\n\to\infty$.
300: In a similar manner we can define the MAP threshold.
301: This threshold was first found
302: via the replica method in \cite{Mon01}.
303: Further, in \cite{Mon01c} a simple counting argument leading to an upper bound
304: for this threshold was given. The argument is explained and sharpened
305: in Sec.~\ref{sec:Counting}.
306: In this paper we develop the
307: point of view taken in \cite{MeU03b}. The reference quantity is then the extrinsic\footnote{The term
308: extrinsic is used when the observation of the bit itself is ignored, see \cite{BGT93,HOP96}.}
309: entropy, in short EXIT.\footnote{The term \exitentropy, introduced in \cite{teB01},
310: stands for extrinsic (mutual) information
311: transfer. Rather than using mutual information we opted to use entropies which
312: in our setting simply means one minus mutual information. It is natural to
313: use entropy in the setting of the binary erasure channel since the parameter $\ih$ itself
314: represents the channel entropy.}
315: The \exitentropy curve
316: associated to the $i^\text{th}$ variable is a function of the channel entropy
317: and it is defined as $H(X_i \mid Y_{[\n]\setminus\{i\}})$.
318: Hereby, $X_i$ represents the $i^{\text{th}}$
319: input bit and, for $S\subseteq[n]\defas\{1,\dots,\n\}$, $X_S$ represents the $|S|$-tuple of
320: all bits indexed by $S$. For notational simplicity, let us write
321: $X_{\sim i}=X_{[\n]\setminus\{i\}}$ when a single bit is omitted and $X=X_{[\n]}$ for the
322: entire vector. The uniformly averaged quantity
323: $\frac{1}{\n}\sum_{i=1}^\n H(X_i \mid Y_{\sim i})$ is called the \exitentropy function.
324: Recall that if there is a uniform prior on the set of hypotheses
325: then the maximum a posteriori
326: and the maximum likelihood decoding rule are identical.
327: Let $\Phi^\MAP_i=\phi^\MAP_i(Y_{\sim i})$ denote the extrinsic MAP bit
328: estimate (sometimes called extrinsic information) associated to the $i^{\text{th}}$ bit. This can be
329: any sufficient statistics for $X_i$ given $Y_{\sim i}$. Since we
330: deal with binary variables, we can always think
331: of it as the conditional expectation $\phi^\MAP_i(Y_{\sim i})
332: \defas \expectation[X_i|Y_{\sim i}]$.
333: Observe that $H(X_i \mid Y_{\sim i})=H(X_i \mid \Phi^\MAP_i)$.
334: %
335: %*********************************************************************
336: %
337: \subsection{Overview of Results}
338: \label{OverviewSection}
339: %
340: Consider a \ddp $(\ledge, \redge)$
341: and the corresponding sequence of ensembles $\ldpc(\n,\ledge,\redge)$ of increasing length $\n$.
342: Fig.~\ref{fig:exitcurve} shows the asymptotic \exitentropy curve for the
343: regular \ddp $(\ledge(x)=x^2, \redge(x)=x^5)$.
344:
345: \begin{figure}[htp]
346: \centering
347: \setlength{\unitlength}{0.625bp}%
348: \begin{picture}(360,190)
349: \put(0,30)
350: {
351: \put(0,0){\includegraphics[scale=0.625]{EXITmlINTRO}}
352: % coordinates
353: \put(150,-5){\makebox(0,0)[t]{\small{$1$}}}
354: \put(-5,150){\makebox(0,0)[tr]{\small{$1$}}}
355: % MAP
356: \put(73,-5){\makebox(0,0)[tl]{\small{$\ih^\MAP$}}}
357: % BP
358: \put(64,-5){\makebox(0,0)[rt]{\small{$\ih^\BP$}}}
359: \put(58,90){\makebox(0,0)[r]{\small{$\xh^\BP(\ih)$}}}
360: }
361: \put(200,30)
362: {
363: \put(0,0){\includegraphics[scale=0.625]{EXITbpINTRO}}
364: % coordinates
365: \put(150,-5){\makebox(0,0)[t]{\small{$1$}}}
366: \put(-5,150){\makebox(0,0)[tr]{\small{$1$}}}
367: % MAP
368: \put(73,-5){\makebox(0,0)[lt]{\small{$\ih^\MAP$}}}
369: \put(79,90){\makebox(0,0)[l]{\small{$\xh^\MAP(\ih)$}}}
370: % BP
371: \put(64,-5){\makebox(0,0)[rt]{\small{$\ih^\BP$}}}
372: }
373: % caption
374: \put(80,0){\makebox(0,0){{\small{(a)}}}}
375: \put(280,0){\makebox(0,0){{\small{(b)}}}}
376: \end{picture}
377: \caption{BP and MAP \exitentropy curves for the \ddp
378: $(\ledge(x)=x^2, \redge(x)=x^5)$.
379: (a) BP \exitentropy curve $\xh^\BP(\ih)$: its parametric equation is stated
380: in (\ref{equ:exitcurve}). It is zero until $\ih^\BP$
381: at which point it jumps. It further
382: continues smoothly
383: until it reaches one at $\ih=1$.
384: (b) MAP \exitentropy curve $\xh^\MAP(\ih)$. Note that the figure (b) includes also
385: the ``spurious'' branch of Eq. (\ref{equ:exitcurve}). The spurious branch
386: corresponds to unstable fixed points.
387: The MAP threshold is determined by the balance of the two dark
388: gray areas.
389: }
390: \label{fig:exitcurve}
391: \end{figure}
392: Formally, this \exitentropy curve is
393: $\xh^\MAP(\ih)\defas\lim_{\n\rightarrow \infty}\frac{1}{n} \sum_{i=1}^{n} H(X_i \mid Y_{\sim i}(\ih))= \lim_{\n\rightarrow \infty}\frac{1}{n} H(X_i \mid \Phi^\MAP_i)$.
394: Its main
395: characteristics
396: are as follows:
397: the function is zero below the MAP threshold $\ih^\MAP$,
398: it jumps at $\ih^\MAP$ to a non-zero value and
399: continues then smoothly until it reaches one for $\ih=1$.
400: The area under
401: the \exitentropy curve equals the rate of the code, see \cite{AKtB02}. Compare
402: this to the equivalent function of the BP
403: decoder which is also shown in Fig.~\ref{fig:exitcurve}.
404: The BP \exitentropy curve $\xh^\BP(\ih)\defas \lim_{\n\rightarrow \infty}\frac{1}{n} H(X_i \mid \Phi^{\BP}_i)$
405: corresponds to running a BP decoder on a very
406: large graph until the decoder has reached a fixed point. The extrinsic entropy of the bits
407: at this fixed point gives the BP \exitentropy curve.
408: This curve is given in parametric form by
409: \begin{align} \label{equ:exitcurve}
410: \left(
411: \frac{\xl}{\ledge(1-\redge(1-\xl))}, \lnode(1-\redge(1-\xl))
412: \right),
413: \end{align}
414: where $\xl$ indicates the erasure probability of the variable-to-check messages.
415: To see this, note that when transmission takes
416: place over BEC($\ih$), then the BP decoder reaches a fixed point $\xl$ which is given by
417: the solution of the density evolution (DE) equation $\ih \ledge(1-\redge(1-\xl))$.
418: We can therefore express
419: $\ih$ as $\ih(\xl)\defas\frac{\xl}{\ledge(1-\redge(1-\xl))}$.
420: Now the average extrinsic probability
421: that a bit is still erased at the fixed point is equal to $\lnode(1-\redge(1-\xl))$.
422: Note that the BP EXIT curve is the trace of this parametric equation for $\xl$
423: starting at
424: $\xl=1$ until $\xl=\xl^\BP$. This is the critical point
425: and $\ih(\xl^\BP)=\ih^\BP$.
426: Summarizing, the BP \exitentropy curve is zero up to the BP threshold
427: $\ih^\BP$ where it jumps to a non-zero value and then continues
428: smoothly until it reaches
429: one at $\ih=1$.
430: %
431: \begin{figure}[htp]
432: \centering
433: \setlength{\unitlength}{0.625bp}%
434: \begin{picture}(360,190)
435: \put(0,30)
436: {
437: \put(0,0){\includegraphics[scale=0.625]{EXITguesses}}
438: % coordinates
439: \put(150,-5){\makebox(0,0)[t]{\small{$1$}}}
440: \put(-5,150){\makebox(0,0)[tr]{\small{$1$}}}
441: % MAP
442: \put(64,-5){\makebox(0,0)[tr]{\small{$\ih^\BP$}}}
443: % BP
444: \put(73,-5){\makebox(0,0)[lt]{\small{$\ih^\MAP$}}}
445: }
446: \put(200,30)
447: {
448: \put(0,0){\includegraphics[scale=0.625]{EXITcontradictions}}
449: % coordinates
450: \put(150,-5){\makebox(0,0)[t]{\small{$1$}}}
451: \put(-5,150){\makebox(0,0)[tr]{\small{$1$}}}
452: % MAP
453: \put(64,-5){\makebox(0,0)[rt]{\small{$\ih^\BP$}}}
454: % BP
455: \put(73,-5){\makebox(0,0)[lt]{\small{$\ih^\MAP$}}}
456: }
457: % caption
458: \put(80,0){\makebox(0,0){{\small{(a)}}}}
459: \put(280,0){\makebox(0,0){{\small{(b)}}}}
460: \end{picture}
461: \caption{
462: %MAP and BP \exitentropy curves for the \ddp
463: %$(\ledge(x)=x^2, \redge(x)=x^5)$.
464: Balance of areas for the Maxwell decoder
465: between the number of guesses in (a) and the number of contradictions in (b).
466: The two dark gray areas are equal at the
467: MAP threshold. These two areas differ from the areas indicated in
468: Fig.~\ref{fig:exitcurve} only by a common part. }
469: \label{fig:exitbalance}
470: \end{figure}
471:
472: In \cite{MeU03b} it was pointed out that
473: for the investigated cases the following two curious relationships between these two
474: curves hold:
475: First, the BP and the MAP curve coincide above $\ih^\MAP$.
476: Second, the MAP curve can be constructed from the %iterative
477: BP curve in the following way.
478: If we draw the BP curve as parameterized in (\ref{equ:exitcurve}) not only
479: for $\xl \in [\xl^\BP,1]$ but also
480: for $\xl \in (0,\xl^\BP)$ we get the
481: curve shown in the right picture of Fig.~\ref{fig:exitcurve}.
482: Notice that the branch for $\xl \in (0,\xl^\BP)$ corresponds to unstable
483: fixed points under BP decoding. Moreover, the fraction of erased
484: messages $\xl$ decreases along this branch when the erasure probability
485: is increased and it satisfies $\ih(\xl)>\ih$.
486: Because of these peculiar features, it is usually
487: considered as ``spurious''.
488: %
489: To determine the MAP threshold take a vertical line at
490: $\ih=\ih^\BP$ and shift it to the right until the area
491: which lies to the left of this line and is enclosed by the line
492: and the BP \exitentropy curve is equal to the area which lies to the right
493: of the line and is enclosed by the line and the BP \exitentropy curve
494: (these areas are indicated in dark gray in
495: the picture).
496: This unique point determines the MAP threshold.
497: The MAP \exitentropy curve is now the curve which is zero to the left of the
498: threshold and equals the iterative curve to the right
499: of this threshold. In other words, the MAP threshold is determined by a balance
500: between two areas.
501: It turns out that there is an operational meaning to this
502: balance condition. We define the so-called Maxwell (M) decoder which performs
503: MAP decoding by combining BP decoding with guessing. The dark gray areas
504: in in the right picture of
505: Fig.~\ref{fig:exitbalance} differ from the ones in Fig.~\ref{fig:exitcurve}
506: only by a common part.
507: We can show that the gray area
508: on the left is connected to
509: the number of ``guesses'' the \maxwell
510: decoder has to venture, while
511: the gray area on the right represents the
512: number of ``confirmations'' regarding these guesses.
513: The MAP threshold is determined by the condition that the number of
514: confirmations balances the number of guesses (i.e., that each guess
515: is confirmed), and therefore the two areas
516: are equal: in other words, at the MAP threshold (and below)
517: there is just a single codeword compatible
518: with the channel received bits.
519: \begin{floatingfigure}{152bp}
520: %\begin{figure}[hbt]
521: \vspace{25bp}
522: \centering
523: \setlength{\unitlength}{0.5bp}
524: \begin{picture}(105,153)
525: \put(-45,0){\includegraphics[scale=0.62]{EXITconclusion}}
526: \put(36,-7){\makebox(0,0){\small{$\ih^\MAP$}}}
527: \put(7,-7){\makebox(0,0){\small{$\ih^\BP$}}}
528: \put(116,88){\makebox(0,0){\small{$\xh^\MAP(\ih)$}}}
529: \put(46,105){\makebox(0,0){\small{$\xh^\BP(\ih)$}}}
530: \end{picture}
531: \caption{\label{fig:multijump} BP (dashed and solid line) and MAP (thick solid line) \exitentropy curves for the
532: ensemble discussed in Examples \ref{equ:doublejumpexample} and \ref{ex:running2jumps}.
533: Both curves have two jumps. The two jumps of the MAP \exitentropy curve are both
534: determined by a local balance of areas.}
535: \vspace{5bp}
536: %\end{figure}
537: \end{floatingfigure}
538:
539: The \exitentropy curves depicted in Fig.~\ref{fig:exitcurve}
540: are representative for a large family of degree distributions, e.g., those of
541: regular LDPC ensembles.
542: But more complicated scenarios are possible.
543: Fig.~\ref{fig:multijump} depicts a slightly more general case
544: in which the BP \exitentropy curve and the MAP \exitentropy curve have two jumps.
545: As can be seen from this figure, the same kind of balance condition
546: holds in this case {\em locally} and it
547: determines the position of each jump.
548:
549: %\subsection{Plan of the Paper}
550: \subsection{Paper Outline}
551:
552: We start by considering the conditional entropy $H(X \mid Y)$,
553: where $X$ is the transmitted
554: codeword and $Y$ the received sequence, and we derive the so-called
555: Area Theorem for finite-length codes.
556: When applying the Area Theorem to the binary erasure channel,
557: the notion of \exitentropy curve enters explicitly.
558: Next, we show that when the codes are chosen randomly
559: from a suitable defined ensemble then the
560: individual conditional entropies and \exitentropy curves concentrate
561: around their ensemble averages. This is the first step towards
562: the asymptotic analysis.
563:
564: We continue by defining the three asymptotic
565: \exitentropy curves of interest. These are
566: the (MAP) \exitentropy curve, the BP \exitentropy curve, and the EBP
567: \exitentropy curve (which holds extended BP \exitentropy and includes the spurious branch).
568: We show that the Area Theorem
569: remains valid in the asymptotic setting. As an immediate consequence
570: we will see that for some classes of ensembles (roughly
571: those for which the stability condition determines the threshold)
572: BP decoding coincides with MAP decoding.
573:
574: We then present a key point of the paper, which is
575: the derivation of an upper-bound for the MAP threshold. Several examples
576: illustrate this technique and lead to suggests the tightness of the bound.
577:
578: The same result is recovered through a counting argument that,
579: supplemented by a combinatorial calculation, implies the tightness of
580: the bound.
581:
582: Finally, we introduce the so-called \maxwell decoder which
583: provides a unified framework for understanding the
584: connection between the BP and the MAP decoder. A closer
585: analysis of the performance of the \maxwell decoder will allow us
586: to prove a refined upper bound on the MAP threshold
587: and it will give rise to a pleasing interpretation
588: of the MAP threshold as that parameter in which two areas under the
589: EBP \exitentropy curve are in balance.
590:
591: We conclude the paper by discussing some applications of our method.
592: %For example, it is proved that
593: %regular LDPC ensembles of increasing degree and fixed rate have
594: %a BP threshold which goes to $0$ whereas their MAP threshold goes to
595: %the Shannon threshold.
596: %
597: %**********************************************************************
598: %
599: \section{Finite-Length Codes: Area Theorem and Concentration}
600:
601: Let $X$ be the transmitted codeword and let $Y$ be the received word.
602: The conditional entropy $H(X \mid Y)$ is
603: % ESSAYER D'ENLEVER LES "clearly" "straightforward" etc
604: %clearly
605: of fundamental importance if we
606: consider the question whether reliable communication is possible. Let us
607: see how this quantity appears naturally in the context of decoding.
608: To this end,
609: we first recall the original Area Theorem
610: as introduced in \cite{AKtB02}.
611: %by Ashikhmin, Kramer and ten Brink \cite{AKtB02}.
612: \btheo[Area Theorem]
613: \label{theo:areatheorem0}
614: Let $X$ be a binary vector of length $n$ chosen with probability $p_X(x)$ from a finite set.
615: Let $Y$ be the result of passing $X$ through BEC$(\cp)$.
616: Let $\Omega$ be a further observation of $X$ so that
617: $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
618: To emphasize that $Y$ depends on the channel parameter $\cp$ we write $Y(\cp)$. Then
619: \begin{align}
620: \label{equ:areatheorem0}
621: \frac{H(X \mid \Omega)}{\n} =
622: \int_0^1\frac{1}{\n}\sum_{i\in[\n]} H(X_i \mid Y_{\sim i}(\cp), \Omega)\text{d}\cp.
623: \end{align}
624: \etheo
625:
626: The reader familiar with the original statement in \cite{AKtB02} will
627: have noticed that we have rephrased the theorem. First,
628: we expressed the result in terms of entropy instead of mutual information.
629: Second, the observations $Y$ and $\Omega$ represent what in the original theorem
630: were called the ``extrinsic'' information and the ``channel,'' respectively.
631:
632: In (\ref{equ:areatheorem0})
633: the integration ranges from zero (perfect channel)
634: to one (no information conveyed). The following is a trivial extension. %an easy generalization.
635: \btheo[Area Theorem]
636: \label{theo:areatheorem1}
637: Let $X$ be a binary vector of length $\n$ chosen with probability $p_X(x)$ from a finite set.
638: Let $Y$ be the result of passing $X$ through BEC$(\cp)$.
639: Let $\Omega$ be a further observation of $X$ so that
640: $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$. Then
641: \begin{align*}
642: \frac{H(X \mid Y(\cp^*), \Omega)}{\n} =
643: \int_0^{\cp^*}\frac{1}{\n}\sum_{i\in[\n]}H(X_i \mid Y_{\sim i}(\cp), \Omega)\text{d}\cp.
644: \end{align*}
645: \etheo
646:
647: {\em Proof of Theorem~\ref{theo:areatheorem1}:}
648: Let $Y^{(1)}$ be the result of passing $X$ through BEC($\cp$)
649: and $Y^{(2)}$ be the result of passing $X$ through BEC($\cp^*$).
650: Let $\Omega$ be the additional observation of $X$. Applying Theorem~\ref{theo:areatheorem0},
651: with $Y=Y^{(1)}$ and with additional observation $(Y^{(2)}, \Omega)$, we
652: have $p_{\Omega, Y^{(2)} \mid X, Y^{(1)}}(\omega, y^{(2)} \mid x, y^{(1)}) =
653: p_{\Omega, Y^{(2)} \mid X}(\omega, y^{(2)} \mid x)$, as required, so that we get
654: \begin{align*}
655: H(X \mid Y^{(2)}(\cp^*), \Omega) =
656: \int_0^1\sum_{i\in[\n]}
657: H(X_i \mid Y^{(1)}_{\sim i}(\cp), Y^{(2)}(\cp^*), \Omega)\text{d}\cp.
658: \end{align*}
659: Now note that
660: \begin{align*}
661: H(X_i \mid Y^{(1)}_{\sim i}(\cp), Y^{(2)}(\cp^*), \Omega) & =
662: \cp^* H(X_i \mid Y_{\sim i}(\cp \cp^*), \Omega).
663: \end{align*}
664: This is true since the bits of $Y^{(1)}_{\sim i}(\cp)$ and $Y^{(2)}(\cp^*)$
665: are erased independently (so that the respective erasure probabilities multiply) and since $Y^{(2)}(\cp^*)$
666: contains the intrinsic observation of bit $X_i$, which is erased with probability $\cp^*$.
667: If we now substitute the right hand side of the last expression in our previous integral and
668: make the change of variables $\cp' = \cp \cdot \cp^*$,
669: Theorem \ref{theo:areatheorem1} %the claim
670: follows.
671: \qed
672:
673: Assume that we allow each $X_i$ to be passed through a different channel BEC$(\cp_i)$.
674: Rather than phrasing our result specifically for the
675: case of the BEC$(\cp_i)$, let us
676: state the area theorem right away in its general form
677: as introduced in \cite{MMRU04}.
678: In this paper we will only be interesting
679: in the consequences as they pertain to transmission over the BEC$(\cp)$.
680: The investigation of the general case is relegated to the companion paper
681: \cite{MMRU05}.
682:
683: In order to state this and subsequent results in a more compact form we introduce the
684: following definition.
685: %
686: \bdefi[Channel Smoothness]
687: Consider a family of memoryless channels with input and output alphabets ${\cal X}$
688: and ${\cal Y}$, respectively, and characterized by their transition
689: probability distribution functions (pdf's)
690: $p_{Y \mid X}(y \mid x)$. If ${\cal Y}$ is discrete, we interpret
691: $p_{Y \mid X}(\cdot \mid x)$ as a pdf with respect to the counting measure.
692: If ${\cal Y}$ is continuous, $p_{Y \mid X}(y \mid x)$ is a density with respect
693: to Lebesgue measure.
694: Assume that the family is parameterized by $\cp$,
695: where $\cp$ takes values in some interval $I \subseteq {\mathbb R}$.
696: The channel is said to be {\em smooth}
697: with respect to the parameter $\cp$
698: if the pdf's $\{p_{Y \mid X}(y \mid x): x \in {\cal X}, \, y \in{\cal Y}\}$
699: are differentiable functions of $\cp \in I$.
700: \edefi
701: %
702: Notice that, if a channel family is smooth, then several basic properties
703: of the channel are likely to be differentiable with respect to the channel
704: parameter. A basic (but important) example is the channel conditional
705: entropy $H(Y|X) = \expectation[-\log \{p_{Y \mid X}(Y \mid X)\}]$
706: given a reference measure $p_{X}(x)$ on ${\cal X}$. Suppose that
707: ${\cal Y}$ is finite, and that, for any $\cp\in I$, $p_{Y|X}(y|x)>0$
708: for any $x \in {\cal X}, \, y \in{\cal Y}$. Then
709: %
710: \begin{align*}
711: %
712: \frac{\de H(X|Y)}{\de \cp} &= \sum_{x,y}p_X(x)\log \left(
713: \frac{1}{p_{Y|X}(y|x)}\right)\,
714: \frac{\de p_{Y|X} }{\de \cp}(y|x)\, .
715: %
716: \end{align*}
717: %
718: In other words, differentiability of $H(Y|X)$ follows from differentiability
719: of $p_{Y|X}(y|x)$ and of $-x\log x$.
720: In this paper we consider families of binary erasure channels which are trivially
721: smooth with respect to the parameter $\cp$.
722:
723: \btheo[General Area Theorem-\cite{MMRU04}]
724: \label{theo:generalareatheorem}
725: Let $X$ be a binary vector of length $n$ chosen with probability $p_X(x)$ from a finite set.
726: Let the channel from $X$ to $Y$ be memoryless, where
727: $Y_i$ is the result of passing $X_i$ through a smooth channel with parameter $\cp_i$, $\cp_i \in I_i$.
728: Let $\Omega$ be a further observation of $X$ so that
729: $p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
730: Then
731: \begin{align}
732: \label{eq:generalareatheorem}
733: \text{d} H(X \mid Y,\Omega) & =
734: \sum_{i=1}^\n \frac{\partial H(X_i \mid Y,\Omega ) }{\partial \cp_i} \text{d}\cp_i.
735: \end{align}
736: \etheo
737: \bproof
738: For $i \in [\n]$, the entropy rule gives $H(X \mid Y,\Omega)=H(X_i \mid Y,\Omega)+H(X_{\sim i} \mid X_i,Y,\Omega)$.
739: %We claim that
740: %\begin{align*}
741: %\label{equ:factorization}
742: We have
743: $p_{X_{\sim i} \mid X_i,Y,\Omega} = p_{X_{\sim i} \mid X_i,Y_{\sim i},\Omega}$ %,
744: %\end{align*}
745: %which is true
746: since the channel is memoryless and $p_{\Omega \mid X, Y} = p_{\Omega \mid X}$. %$p_{\Omega \mid X, Y}(\omega \mid x, y) = p_{\Omega \mid X}(\omega \mid x)$.
747: Therefore, $H(X_{\sim i} \mid X_i,Y,\Omega)=H(X_{\sim i} \mid X_i,Y_{\sim i},\Omega)$ and
748: $\frac{\partial H(X \mid Y,\Omega ) }{\partial \cp_i} = \frac{\partial H(X_i \mid Y,\Omega) }{\partial \cp_i}$.
749: From this the total derivate as stated in
750: (\ref{eq:generalareatheorem}) follows
751: immediately.
752: \eproof
753:
754: {\em{\hspace{16pt} Alternative proof of Theorem~\ref{theo:areatheorem1}:}}
755: Keeping in mind that transmission takes place over a binary erasure channel, we write
756: %
757: \begin{align*}
758: %
759: H(X_i \mid Y, \Omega)&= \sum_{y_i\in\{\0,\?,\1\}}p_{Y_i}(y_i)%\Pr(Y_i=y_i)\,
760: H(X_i\mid Y_i=y_i,Y_{\sim i}, \Omega)\, .
761: %
762: \end{align*}
763: %
764: The terms corresponding to $y_i\in\{\0,\1\}$ vanish because $X_i$ is then completely
765: determined by the channel output. The remaining term yields
766: $H(X_i \mid Y, \Omega) = \epsilon_i H(X_i\mid Y_{\sim i}, \Omega)$,
767: %\label{equ:becentropyformula}
768: %
769: because $p_{Y_i}(\?) =\epsilon_i$, and the occurrence at the channel output of an erasure
770: at position $i$ is independent from $X$, $Y_{\sim i}$ and $\Omega$.
771: %Assume now that $\cp_i=\cp$ for all $i \in [n]$.
772: We can then write %Then
773: \begin{align*}
774: \text{d} H(X \mid Y (\cp),\Omega) & =
775: \sum_{i \in [\n]} \frac{\partial H(X_i \mid Y,\Omega ) }{\partial \cp_i} \text{d} \cp_i \\
776: & = \sum_{i \in [\n]} H(X_i \mid Y_{\sim i},\Omega) \text{d} \cp_i,
777: %\tag*{by (\ref{equ:becentropyformula}).}%~\text{\qed}}
778: \end{align*}
779: %By restricting at this point the individual channels $\{$BEC($\ih_i$)$\}$
780: %to be parametrized by a common parameter $\cp$,
781: %Theorem~\ref{theo:areatheorem1} follows.
782: %\qed
783: which, when we assume that $\cp_i=\cp$ for all $i \in [n]$, gives Theorem~\ref{theo:areatheorem1}.\qed
784:
785: A few remarks are in order.
786: First, the additional degree of freedom afforded by allowing
787: an extra observation $\Omega$ is useful when studying the
788: dynamical behavior of certain iterative coding schemes via
789: \exitentropy chart arguments. %when employing EXIT chart arguments for
790: %concatenated ensembles.
791: (For example, in a parallel concatenation,
792: $Y$ typically represents the observation of the systematic bits
793: and $\Omega$ represents the fixed channel observation of the
794: parity bits.)
795: For the purpose of this paper however, the additional observation
796: $\Omega$ is not needed since we are not concerned by componentwise
797: \exitentropy charts. We will therefore skip $\Omega$ in the sequel.
798: %
799: Second, as emphasized in the last step in the previous proof, we
800: can assume at this point, more generally, that the individual channel
801: parameters $\cp_i$ are not the same but that the individual
802: channels are all
803: parametrized by a common parameter $\cp$.
804: For instance one may think of a families $\{$BEC($\ih_i$)$\}$
805: where $\ih_i(\ih)$ are smooth functions of $\ih\in[0,1]$.
806: In the simplest case some parameter might be chosen to be constant.
807: This degree of freedom allows for an elegant proof of
808: Theorem \ref{theo:ebpareatheorem}.
809:
810: One of the main aims of this paper is to investigate the MAP performance
811: of sparse graph codes in the limit of large blocklengths. Our task is
812: made much easier by realizing that we can restrict our study to the
813: {\em average} such performance. More precisely, let $\graph$ be chosen
814: uniformly at random from $\ldpc(\ledge,\redge,\n)$ and let $H_\graph(X
815: \mid Y)$ denote the conditional entropy for the code $\graph$.
816: We state the following theorems right away
817: for general binary memoryless symmetric (BMS) channels.
818: \btheo[Concentration of Conditional Entropy]
819: \label{theo:concentrationml}
820: Let $\graph$ be chosen uniformly at random from $\ldpc(\n,\ledge,\redge)$.
821: Assume that $\graph$ is used to transmit over a BMS channel.
822: By some abuse of notation, let
823: $H_{\graph(n)} =H_\graph(X \mid Y)$ be the associated conditional entropy.
824: Then for any $\xi > 0$
825: %
826: \begin{align*}
827: \Pr \left\{|H_{\graph(n)}-\expectation\left[H_{\graph(n)}\right]|>\n \xi\right\} &\leq 2\,
828: \text{e}^{-\n B \xi^2},\\
829: \end{align*}
830: where $B= 1/(2 (\drmax+1)^2(1-r))$ and where $\drmax$ is the maximal check-node degree.
831: \etheo
832: %
833: \bproof
834: The proof uses the standard technique of first constructing a Doob's martingale
835: with bounded differences and then applying the Hoeffding-Azuma inequality.
836: The complete proof can be found in
837: \cite{Mon04} and it is reported in an adapted and streamlined form in
838: Appendix~\ref{app:ConcentrationProofs}.
839: \eproof
840: Let us now consider the concentration of the MAP \exitentropy curve.
841: For the BEC this curve is given equivalently by
842: $\frac{1}{\n} \sum_{i=1}^\n H_{\graph(n)}(X_i \mid Y_{\sim i}(\ih))$ or
843: by $\frac{1}{\n} H_{\graph(n)}'(X \mid Y(\ih))$. We
844: choose the second representation and phrase the statement in terms
845: of the derivative of the conditional entropy with respect
846: to the channel parameter $\ih$.
847: \btheo[Concentration of MAP \exitentropy Curve]
848: \label{theo:concentrationexitml}
849: Let $\graph$ be chosen uniformly at random from $\ldpc(\n,\ledge,\redge)$
850: and let $\{\text{BMS}(\cp)\}_{\cp \in I}$ denote a family of BMS channels ordered by physical degradation (with $\text{BMS}(\cp')$ physically degraded
851: with respect to $\text{BMS}(\cp)$ whenever $\cp'>\cp$)
852: and smooth with respect to $\cp$.
853: Assume that $\graph$ is used to transmit over the BMS$(\cp)$ channel.
854: Let $H_{\graph(n)} =H_\graph(X \mid Y)$ be the associated conditional entropy.
855: Denote by $H'_{\graph(n)}$ the derivative of
856: $H_{\graph(n)}$ with respect to $\cp$ (such a derivative exists because of the explicit
857: calculation presented in Theorem \ref{theo:generalareatheorem}) and
858: let $J \subseteq I$ be an interval on which
859: $\lim_{n\to\infty} \frac{1}{n}\expectation \left[H_{\graph(n)}\right]$
860: exists and is differentiable with respect
861: to $\cp$. Then, for any $\epsilon\in J$ and $\xi>0$ there exist an
862: $\alpha_{\xi}>0$ such that, for $n$ large enough
863: \begin{align*}
864: \Pr \left\{|H'_{\graph(n)}-\expectation [H'_{\graph(n)} ]|>\n \xi\right\} &\leq
865: \text{e}^{-\n \alpha_{\xi}}.
866: \end{align*}
867: Furthermore, if
868: %$\xh^\MAP(\cp)$
869: $\lim_{n\to\infty} \frac{1}{n}\expectation \left[H_{\graph(n)}\right]$
870: is twice differentiable with respect
871: to $\cp\in J$, there exists a strictly positive constant $A$ such that $\alpha_\xi>A\xi^4$.
872: \etheo
873: The proof is deferred once more to Appendix~\ref{app:concentrationproofs}.
874:
875: Notice the two extra hypothesis with respect to
876: Theorem~\ref{theo:concentrationml}. First, we assumed that the channel
877: family $\{\text{BMS}(\cp)\}_{\cp \in I}$ is ordered by physical
878: degradation. This ensures that $H'_n$ is non-negative. This condition
879: is trivially satisfied for the family $\{\text{BEC}(\cp)\}_{\cp \in
880: [0, 1]}$. More generally, we can let $\cp$ be any function of the
881: erasure probability differentiable and increasing
882: from zero to one. The second condition, namely the
883: existence and differentiability of the expected entropy per bit in the
884: limit, is instead crucial. As discussed in the previous section (see, e.g.,
885: Fig.~\ref{fig:exitcurve}), the asymptotic \exitentropy curve may have jumps. By
886: Theorem~\ref{theo:areatheorem1} these jumps correspond to discontinuities
887: in the derivative of the conditional entropy. At a jump $\cp_*$, the
888: value of the \exitentropy
889: curve may vary dramatically when passing from one element of the ensemble
890: to the other. Some (a finite fraction) of the codes will perform well,
891: and have an \exitentropy curve close to the asymptotic value at $\cp_*-\delta$,
892: while others (a finite fraction) may have an \exitentropy function close to the
893: asymptotic value at $\cp_*+\delta$ ($\delta$ is here a generic small
894: positive number).
895:
896: \btheo[Concentration of BP \exitentropy Curve]
897: \label{theo:concentrationexitbp}
898: Let $\graph$ be chosen uniformly at random from $\ldpc(\n,\ledge,\redge)$.
899: Assume that $\graph$ is used to transmit over a BMS channel and let
900: $\Phi_i^{\BP, \iter}= \phi_i^{\BP, \iter}(Y_{\sim i})$ denote the
901: extrinsic estimate (conditional mean)
902: of $X_i$ produced by the BP decoder after $\iter$ iterations.
903: Denote by $H_{\graph,i}^{\BP, \iter} =
904: H_\graph \bigl(X_i \mid \Phi_i^{\BP, \iter} \bigr)$ the
905: resulting (extrinsic) entropy of the binary variable $X_i$.
906: Then, for all $\xi>0$, there exists $\alpha_\xi>0$, such that
907: \begin{multline}
908: \Pr\Bigl\{\Bigl|\sum_{i=1}^\n \Bigl(H_{\graph,i}^{\BP, \iter} -
909: \expectation_\graph \Bigl[ H_{\graph,i}^{\BP, \iter} \Bigr] \Bigr)\Bigr|> n \xi\Bigr\}
910: \leq \text{e}^{-\alpha_\xi \n}.
911: \end{multline}
912: \etheo
913:
914: \bproof
915: The proof is virtually identical to the ones given in \cite{LMSS01,RiU01} where the
916: probability of decoding error is considered.
917: \eproof
918: %
919: %**************************************************************************
920: %
921:
922: \section{Asymptotic Setting}
923:
924: \subsection{(MAP) \exitentropy}
925:
926: The next definition and theorem define our main object of study.
927: \bdefi \label{def:asymptoticexit}
928: Let ${\cal C}(n)$ be a sequence of code ensembles of diverging blocklength $\n$
929: and let $\graph(n)$ be chosen uniformly at random from ${\cal C}(n)$.
930: Assume that $\lim_{n \rightarrow \infty} \expectation_{\graph}\left[\frac{1}{\n}
931: \sum_{i=1}^\n H_{\graph(n)}'(X \mid Y(\ih))\right]$ exists.
932: Then this limit is called the asymptotic \exitentropy function
933: of the family of ensembles and
934: we denote it by $\xh^\MAP(\ih)$. We define the MAP threshold
935: $\cp^\MAP$ to be the supremum of all values $\cp$ such that $\xh^\MAP(\cp) = 0$.
936: \edefi
937: Given a \ddp $(\ledge,\redge)$, consider the sequence of
938: ensembles $\{\ldpc(\ledge,\redge,\n)\}_{\n}$.
939: %Let $\graph$ be chosen
940: %uniformly at random from $\ldpc(\ledge,\redge,\n)$.
941: It is natural to conjecture that the associated asymptotic
942: \exitentropy function exists.
943: Note that from Theorem \ref{theo:concentrationexitml} we know
944: that if this limit exists, then individual code
945: instances are closely concentrated around the ensemble average.
946: It is therefore meaningful to define in such a setting the
947: MAP threshold in terms of the ensemble average.
948:
949: Unfortunately, no {\em general} proof of the existence of the
950: MAP \exitentropy curve is known.
951: But we will show how one
952: can in {\em most cases} compute the asymptotic \exitentropy function explicitly for a given ensemble,
953: thus proving existence of the limit in such cases.
954: See also \cite{ZC95} for a discussion on asymptotic thresholds.
955:
956: It is worth pointing out that we defined the MAP threshold to be
957: the channel parameter at which the conditional entropy becomes sublinear.
958: At this point the average conditional bit entropy converges to zero, so
959: that this point is the bit MAP threshold. We note that for some ensembles the
960: block MAP threshold is strictly smaller than the bit MAP threshold.
961:
962:
963: \btheo[Asymptotic Area Theorem]
964: \label{theo:asymptoticareatheorem}
965: Consider a \ddp $(\ledge,\redge)$. Assume that the associated asymptotic \exitentropy function
966: as defined in Definition~\ref{def:asymptoticexit} exists for all
967: $\cp\in[0,1]$.
968: Assume further that the limit
969: $\drate_{\text{as}} =
970: \lim_{\n \rightarrow \infty}\expectation_{\graph} \Bigl[\frac{H(X)}{\n} \Bigr]$
971: exists.
972: Then
973: \begin{align*}
974: \drate_{\text{as}} & =
975: \int_0^{1} \xh^\MAP(\ih)\mbox{d} \ih.
976: \end{align*}
977: \etheo
978: \bproof
979: Let $\xh^\MAP_{\graph(\n)}(\ih)$ denote the \exitentropy function associated to a particular
980: $\graph \in\ldpc(\ledge,\redge,\n)$ with rate $r_{\graph(\n)}$.
981: % By definition, $\xh^\MAP(\ih)=\lim_{\n\to\infty}\expectation_{\graph}
982: %\left[\xh^\MAP_{\graph(\n)}(\ih)\right]$.
983: We have
984: \begin{align}
985: \notag \int_0^1 \expectation_{\graph}\left[\xh^\MAP_{\graph(\n)}(\ih)\right] \text{d}\ih & =\expectation_{\graph}\left[\int_0^1 \xh^\MAP_{\graph(\n)}(\ih) \text{d}\ih \right] \\
986: \label{eq:fubbini} & = \expectation_{\graph}\Bigl[\frac{H(X)}{\n}\Bigr] ~~{\underset{\n \to \infty }{\longrightarrow}}~~ \drate_{\text{as}}
987: \end{align}
988: The first equality is obtained by noticing that
989: the function $\xh^\MAP_{\graph(\n)}(\ih)$ is non-negative. We are therefore
990: justified by Fubini theorem to switch the order of integration.
991: The second step follows from the Area Theorem (the rate being equal to $\frac{H(X)}{\n}$).
992:
993: On the other hand, the Dominated Convergence Theorem can be applied to the sequence
994: $\left\{\expectation_{\graph}\left[\xh^\MAP_{\graph(\n)}(\ih)\right]\right\}_{\n}$ since it
995: converges (as assumed in the hypothesis) to $\xh^\MAP(\ih)$
996: and is trivially upper-bounded by 1. We therefore get
997: \begin{align}
998: \notag \lim_{\n \to \infty} \int_0^1 \expectation_{\graph}\left[\xh^\MAP_{\graph(\n)}(\ih)\right]
999: \text{d}\ih &= \int_0^1 \lim_{\n \to \infty}
1000: \expectation_{\graph}\left[\xh^\MAP_{\graph(\n)}(\ih)\right] \text{d}\ih\\
1001: \notag & = \int_0^1 \xh^\MAP(\ih) \text{d} \ih.
1002: \end{align}
1003: which, combined with (\ref{eq:fubbini}), concludes the proof.
1004: \eproof
1005:
1006: Lemma \ref{WELemma} gives a sufficient condition for the limit
1007: $\drate_{\text{as}}$ to exists. Note that under this condition
1008: the asymptotic rate $\drate_{\text{as}}$ is equal to
1009: to the design rate $\drate(\ledge,\redge)$.
1010: Most \ddps $(\ledge,\redge)$ encountered in practice fulfill this condition.
1011: This condition is therefore not
1012: very restrictive.
1013:
1014: %
1015: %***********************************************************************
1016: %
1017: \subsection{BP \exitentropy} \label{sec:bpexitfunction}
1018: Recall that the MAP \exitentropy curve can be expressed as
1019: $H(X_i \mid \Phi^\MAP_i)$ where $\Phi^\MAP_i = \phi_i^\MAP(Y_{\sim i})$
1020: is the posterior estimate (conditional mean) of $X_i$ given $Y_{\sim i}$.
1021: Unfortunately this quantity is not easy to evaluate.
1022: In fact, the main aim of this paper is to accomplish this task.
1023:
1024: %\begin{figure}[htp]
1025: \begin{floatingfigure}{115bp}
1026: %\begin{center}
1027: %\vspace{10bp}
1028: \centering
1029: \setlength{\unitlength}{0.5bp}
1030: \begin{picture}(210,230)
1031: \put(19,10)
1032: {
1033: \put(0,0){\includegraphics[scale=0.62]{bpexit}}
1034: \setlength{\unitlength}{0.62bp}
1035: \put(7,177){\makebox(0,0){\small{$\xh^\BP$}}}
1036: \put(170,0){\makebox(0,0){\small{$\ih$}}}
1037: \put(-8,-8){\makebox(0,0){\small{$0$}}}
1038: \put(153,-8){\makebox(0,0){\small{$1$}}}
1039: \put(-8,154){\makebox(0,0){\small{$1$}}}
1040: }
1041: \end{picture}
1042: %\end{center}
1043: \caption{\label{fig:bpexit} BP \exitentropy function $\ih \mapsto \xh^\BP(\ih)$.}
1044: \vspace{2bp}
1045: \end{floatingfigure}
1046: %\end{figure}
1047: A related quantity which is much easier to compute is
1048: the BP \exitentropy curve shown in Fig.~\ref{fig:bpexit}
1049: for the \ddp $(\xl^2,\xl^5)$. The BP \exitentropy corresponds to
1050: $H(X_i \mid \Phi_i^\BP)$, where $\Phi_i^\BP = \phi_i^\BP(Y_{\sim i})$
1051: is the extrinsic estimate of $X_i$ delivered by the BP decoder
1052: Here a fixed number of iterations, let us say $\iter$, is understood.
1053: Asymptotically, we consider $\iter\to\infty$ {\em after} $\n\to\infty$.
1054: An exact expression for the average asymptotic BP \exitentropy curve for LDPC ensembles
1055: is easily computed via the DE method \cite{LMSS01,LMSS01b,RiU01,RSU01}.
1056:
1057: %
1058: \begin{figure}[htp]
1059: \centering
1060: \setlength{\unitlength}{1bp}
1061: \begin{picture}(240,140)
1062: \put(5,28)
1063: {
1064: \put(5,5){\includegraphics[width=100bp]{EPSwith2J}}
1065: \put(135,5){\includegraphics[width=100bp]{BPEXITwith2J}}
1066: % coordinates
1067: \put(0,0){\makebox(0,0){\small{$0$}}}
1068: \put(100,0){\makebox(0,0){\small{$1$}}}
1069: \put(0,100){\makebox(0,0){\small{$1$}}}
1070: \put(130,0){\makebox(0,0){\small{$0$}}}
1071: \put(230,0){\makebox(0,0){\small{$1$}}}
1072: \put(130,100){\makebox(0,0){\small{$1$}}}
1073: % epsilon
1074: \put(7,-6){\makebox(0,0){\small{$\lxl^0$}}}
1075: \put(16,-6){\makebox(0,0){\small{$\lxl^1$}}}
1076: \put(28,-5){\makebox(0,0){\small{$\uxl^1$}}}
1077: \put(42,-6){\makebox(0,0){\small{$\lxl^2$}}}
1078: \put(101,-6){\makebox(0,0){\small{$\uxl^2$}}}
1079: \put(0,48){\makebox(0,0){\small{$\ih^1$}}}
1080: \put(0,57){\makebox(0,0){\small{$\ih^2$}}}
1081: \put(60,80){\makebox(0,0){\small{$\ih(\xl)$}}}
1082: % EXIT
1083: \put(178,-5){\makebox(0,0){\small{$\ih^\BP$}}}
1084: \put(177,13){\makebox(0,0){\small{$\ih^1$}}}
1085: \put(180,68){\makebox(0,0){\small{$\ih^2$}}}
1086: \put(200,80){\makebox(0,0){\small{$\xh^\BP(\ih)$}}}
1087: }
1088: % caption
1089: \put(60,7){\makebox(0,0){{\small{(a)}}}}
1090: \put(190,7){\makebox(0,0){{\small{(b)}}}}
1091: \end{picture}
1092: \caption{
1093: BP \exitentropy curve with two discontinuities ($J$=2):
1094: (a) Channel entropy function $\xl\mapsto\ih(\xl)$ (b) BP \exitentropy function $\ih\mapsto\xh^\BP(\ih)$.
1095: This example corresponds to the \ddp $(\ledge,\redge)=(0.3x+0.3x^2+0.4x^{13},x^6)$, which has
1096: design rate $\drate\approx0.48718$.
1097: The BP threshold is $\ih^\BP\approx0.48437$ at $\xl^\BP\approx0.09904$.
1098: This is also the first discontinuity, i.e., $\ih^1=\ih^\BP$, $\lxl^1=\xl^\BP$ and $\uxl^1\approx0.22156$.
1099: The second discontinuity occurs for $\ih=\ih^2\approx0.51553$ at $\lxl^2\approx0.37016$ ($\uxl^1=1$).
1100: }
1101: \label{fig:epsilon}
1102: \end{figure}
1103:
1104: Consider the fixed-point condition for the density evolution equations,
1105: \begin{align*}
1106: \ih \ledge(1-\redge(1-\xl)) & = \xl.
1107: \end{align*}
1108: Solving for $\ih$, we get $\ih(\xl)\defas\frac{\xl}{\ledge(1-\redge(1-\xl))},~\xl\in(0,1]$.
1109: In words, for each non-zero fixed-point $\xl$ of density evolution, there is a unique channel
1110: parameter $\ih$. At this fixed-point the asymptotic average BP \exitentropy function equals
1111: $\lnode(1-\redge(1-\xl))$.
1112: If $\cp(\xl)$ is monotonically increasing in $\xl$ over the whole range $[0, 1]$,
1113: then the BP \exitentropy curve is given in parametric form by
1114: \begin{equation}
1115: \label{equ:bpexitcurve}
1116: \bigl(
1117: \cp(\xl), \lnode(1-\redge(1-\xl)) \bigr).
1118: \end{equation}
1119: For some ensembles (e.g., regular cycle-code ensembles) $\cp(\xl)$ is indeed
1120: monotone increasing over the whole range $[0, 1]$, but for most ensembles this
1121: is not true. In this case we have to restrict the above
1122: parameterization to the unique union of intervals
1123: \[
1124: {\cal{I}}\defas\bigcup_{i\in[J]}[\lxl^i,\uxl^i)\cup\{1\},
1125: \]
1126: which has the property that $\cp(\xl)$ is continuously and monotonically
1127: increasing from $\cp^\BP$ to one as $\xl$ takes on increasing values in ${\cal I}$
1128: and for all $i \in [J]$, $\lxl^i=0$ or $\cp'(\lxl^i)=0$. An example of such a partition is
1129: shown in Fig.~\ref{fig:epsilon}. That such a partition exists and is unique
1130: follows from the fact that $\cp(\xl)$ is a differentiable function for $\xl \in [0, 1]$
1131: as can be verified by direct computation.
1132: Set $\uxl^J=1$ and note that $\cp(1)=1 \geq 0$.
1133: Define $\lxl^J$ as the largest nonnegative value of $\xl \leq \uxl^J$ for which $\cp'(\xl)=0$.
1134: If no such value exists then $\cp(\xl)$ is monotonically increasing over the whole
1135: range $[0, 1]$. In this case $J=1$ and we set $\lxl^J=0$.
1136: Now proceed recursively. Assume that the intervals $[\lxl^{i+1}, \uxl^{i+1})$
1137: have been defined and that $\lxl^{i+1}>0$.
1138: Define $\uxl^i$ as the largest nonnegative value of $\xl < \lxl^{i+1}$
1139: such that $\cp(\xl)=\cp(\lxl^{i+1})$. Note that if such a value exists then
1140: we must have $\cp'(\xl) \geq 0$. If no such value exists then we have already found the sought
1141: after partition and we stop. Otherwise define
1142: $\lxl^i$ as the largest nonnegative value of $\xl \leq \uxl^i$ for which $\cp'(\xl)=0$.
1143: As before, if no such value exists then set $\lxl^i=0$ and stop.
1144: Without loss we can eliminate from the resulting partition any interval of zero length.
1145: Let $J$ denote the number of remaining intervals of nonzero length. Note, if the BP threshold
1146: happens at a discontinuous phase transition (jump), then $\xl^\BP=\lxl^1$
1147: and $\cp^\BP=\cp(\lxl^1)$, otherwise, if the BP threshold is given by the stability condition,
1148: then $\xl^\BP=\lxl^0=0$
1149: and $\ih^\BP=\ih(\lxl^0)$. See also Fig. \ref{fig:epsilonwith1JandStabCond}.
1150: %{\bf Not necessarily, e.g., if there is given by stab: adapt this description, see my previous one}
1151: \bcor
1152: \label{cor:asymptoticbpexit}
1153: Assume we are given a \ddp $(\ledge,\redge)$ and that transmission takes place over the BEC.
1154: Let ${\cal{I}}\defas\bigcup_{i\in[J]}[\lxl^i,\uxl^i)\cup\{1\}$ be the partition associated to
1155: $(\ledge,\redge)$. Define $\cp^\BP=\cp(\lxl^1)$.
1156: Then the BP \exitentropy function $\xh^\BP(\ih)$ is equal to zero for $0 \leq \cp < \cp^\BP$ and
1157: for $\cp > \cp^\BP$ it has the parametric characterization
1158: \begin{align*}
1159: (\cp(\xl),\lnode(1-\redge(1-\xl))),
1160: \end{align*}
1161: where $\xl$ takes on all values in ${\cal I}$.
1162: \ecor
1163:
1164: \bprop[Regular LDPC Ensembles ``Jump'' at Most Once]
1165: \label{fact:regular} Consider the
1166: regular \ddp $(\ledge(x),\redge(x))=(x^{\ldegree-1},x^{\rdegree-1})$. Then the function
1167: $\ih(\xl)\defas\frac{\xl}{\ledge(1-\redge(1-\xl))}$ has a unique minimum in the range $[0, 1]$.
1168: Let $\xl^\BP$ denote the location of this minimum. Then $\ih(\xl)$ is
1169: strictly decreasing on $(0,\xl^\BP)$ %for $\xl \in (0,\xl^\BP)$
1170: and strictly increasing on $(\xl^\BP,1)$. %for $\xl \in (\xl^\BP,1)$.
1171: Moreover, $\xl^\BP=0$ if and only if $\ldegree=2$.
1172: \eprop
1173: \bproof
1174: Note that $\cp(1)=1$ and by direct calculation we see that $\cp'(1)=1$.
1175: Therefore, either $\cp(\xl)$ takes on its minimum value within the interval $[0, 1]$
1176: for $\xl=0$ or its minimum value is in the interior of the region $[0, 1]$.
1177: %
1178: Computing explicitly the derivative of $\cp(\xl)$,
1179: we see that the location of the minima of $\cp(\xl)$ must be a root of
1180: $W(\xl)\defas 1-(1-\xl)^{\rdegree-1}-(\ldegree-1)(\rdegree-1)(1-\xl)^{\rdegree-2}\xl$. Furthermore
1181: $W'(\xl) = -(\rdegree-1)(1-\xl)^{\rdegree-3}\{(\ldegree-2)-
1182: [(\ldegree-1)(\rdegree-1)-1]\xl\}$.
1183: Notice that $W(0) = 0$, $W'(0) = -(\rdegree-1)(\ldegree-2)<0$ and $W(1) = 1$.
1184: By the Intermediate Value Theorem, $W(\xl)$ vanishes at least once in $(0,1)$.
1185: Suppose now that $W(\xl)$ vanishes more than once in $(0,1)$, and consider
1186: the first two such zeros $\xl_1$, $\xl_2$. It follows that
1187: $W'(\xl)$ must vanish at least twice: once in $(0,\xl_1)$ and once
1188: in $(\xl_1,\xl_2)$. On the other end, the above explicit expression
1189: implies that $W'(\xl)$ vanishes just once in $(0,1)$, at
1190: $\xl = (\ldegree-2)/[(\ldegree-1)(\rdegree-1)-1]$.
1191: Therefore $W(\xl)$ has exactly one root in $(0,1)$. See also \cite{BRU04}.
1192: \eproof
1193: A dynamic interpretation of the convergence of the BP decoding when the number of
1194: iterations $\iter\to\infty$
1195: is shown in Appendix \ref{app:gapinterpretation} using component EXIT curves. It is
1196: further shown in Appendix \ref{app:areass} and Theorem \ref{theo:bpintegrationfact}
1197: how to compute the area under the BP EXIT curve.
1198: The calculations show that this area is always
1199: larger or equal the design rate. Moreover, some calculus
1200: reveals that, whenever the BP EXIT function has discontinuities, then the
1201: area is strictly larger than the design rate $\drate$.
1202:
1203: % which is also the area under
1204: %the MAP EXIT curve under some asumptions (when $\drate=r_{\text{as}}$, see Section \ref{sec:Counting}).
1205: % Example \ref{ex:stco} illustrates this observation by showing that the
1206: %BP and MAP threshold might be equal even if their respective \exitentropy functions
1207: %are not pointwise equal.
1208: %\bex\label{ex:stco}
1209: %Consider the \ddp $(\ledge,\redge)=(0.4 x + 0.6 x^6,x^6)$ and the
1210: %corresponding $\ldpc$ ensemble with design rate $\drate=0.5$. Using a weight enumerator
1211: % function, see, e.g., Section \ref{sec:Counting}, one can show that
1212: %$\drate=r_{\text{as}}=\int \xh^\MAP$. A quick look
1213: %shows that the BP threshold is given by the stability condition, i.e.,
1214: %it is $\ih^\BP\approx0.4167$ obtained for $\xl\approx\lxl^0=0$. When the
1215: %parameter is $\uxl^0\approx0.04828$, i.e., at $\ih^1\approx0.4691$, a discontinuity
1216: %of the BP \exitentropy curve appears and the edge erasure
1217: %probability $\xl$ ``jumps'' to $\lxl^1\approx0.3309$. This situation is
1218: %shown in Fig.~\ref{fig:epsilonwith1JandStabCond}. %Note
1219: %%that the simple upper-bound found using the BP \exitentropy would be
1220: %%$\overline{\ih}^\BP \approx 0.4940$ obtained for $\xl \approx 0.4332$.
1221: %\begin{figure}[htp]
1222: %\centering
1223: %\setlength{\unitlength}{1bp}
1224: %\begin{picture}(240,140)
1225: %\put(5,28)
1226: %{
1227: %\put(5,5){\includegraphics[width=100bp]{EPSwith1JandStabCond.ps}}
1228: %\put(135,5){\includegraphics[width=100bp]{EXITwith1JandStabCond.ps}}
1229: %% coordinates
1230: %\put(0,0){\makebox(0,0){\small{$0$}}}
1231: %\put(100,0){\makebox(0,0){\small{$1$}}}
1232: %\put(0,100){\makebox(0,0){\small{$1$}}}
1233: %\put(130,0){\makebox(0,0){\small{$0$}}}
1234: %\put(230,0){\makebox(0,0){\small{$1$}}}
1235: %\put(130,100){\makebox(0,0){\small{$1$}}}
1236: %%% epsilon
1237: %\put(5,-6){\makebox(0,0){\small{$\lxl^0$}}}
1238: %%%\put(14,-6){\makebox(0,0){\small{$\uxl^0$}}}
1239: %%\put(37,-6){\makebox(0,0){\small{$\lxl^1$}}}
1240: %\put(-1,43){\makebox(0,0){\small{$\ih^\BP$}}}
1241: %\put(-1,53){\makebox(0,0){\small{$\ih^1$}}}
1242: %\put(60,80){\makebox(0,0){\small{$\ih(\xl)$}}}
1243: %% EXIT
1244: %%\put(176,-5){\makebox(0,0){\small{$\ih^\BP$}}}
1245: %\put(185,17){\makebox(0,0){\small{$\ih^1$}}}
1246: %\put(200,80){\makebox(0,0){\small{$\xh^\BP(\ih)$}}}
1247: %}
1248: % caption
1249: %\put(60,7){\makebox(0,0){{\small{(a)}}}}
1250: %\put(190,7){\makebox(0,0){{\small{(b)}}}}
1251: %\end{picture}
1252: %\caption{
1253: %BP \exitentropy entropy curve with 1 discontinuity ($J$=1) for which the BP threshold $\ih^\BP=\ih^\MAP$ is given by the stability condition:
1254: %(a) Channel entropy function $\xl\mapsto\ih(\xl)$ (b) BP \exitentropy function $\ih\mapsto\xh^\BP(\ih)$.
1255: %The function $\erasurepch(\erasurepex)$ has a first minimum at $\fpit = \erasurepex^{(1)}=0$ and second one at $\erasurepex^{(3)}=0.0482835$ for which the first discontinuity occurs.
1256: %This example is obtained for the \ddp $(\ledge,\redge)=(0.3x+0.3x^2+0.4x^{13},x^6)$
1257: %with design rate $\drate\approx0.48718$.
1258: %The BP threshold is $\ih^\BP\approx0.48437$ at $\xl^\BP\approx0.09904$.
1259: %This is also the first discontinuity, i.e., $\ih^1=\ih^\BP$, $\lxl^1=\xl^\BP$ and $\uxl^1\approx0.22156$.
1260: %The second discontinuity occurs for $\ih=\ih^2\approx0.51553$ at $\lxl^2\approx0.37016$ ($\uxl^1=1$).
1261: %}
1262: %\label{fig:epsilonwith1JandStabCond}
1263: %\end{figure}
1264: %Since the BP threshold is determined by the
1265: %stability condition,
1266: %as explained in the beginning of the section,
1267: %Section
1268: %\ref{UpperSection},
1269: %we have $\ih^\BP=\ih^\MAP$.
1270: %This is true despite the fact that the integral
1271: %under the BP \exitentropy is larger than $\drate=r_{\text{as}}$!
1272: %\eex
1273:
1274:
1275:
1276:
1277: %
1278: %************************************************************************
1279: %
1280:
1281: \subsection{Extended BP \exitentropy Curve}
1282: %
1283: %Recall that by the Area Theorem the integral under $\xh^\MAP(\ih)$ equals the
1284: %rate of the code. Since the BP decoder is in general suboptimal we must
1285: %have $\xh^\MAP(\ih) \leq \xh^\BP(\ih)$. Therefore the integral under $\xh^\BP(\ih)$
1286: %is in general strictly larger than the asymptotic rate of the code $r_{\text{as}}$.
1287:
1288: \begin{figure}[htp]
1289: %\begin{floatingfigure}{120bp}
1290: %\vspace{10bp}
1291: %\begin{figure}[htp]
1292: \centering
1293: \setlength{\unitlength}{0.5bp}
1294: \begin{picture}(210,230)
1295: \put(19,10)
1296: {
1297: \put(0,0){\includegraphics[scale=0.62]{ebpexit}}
1298: \setlength{\unitlength}{0.62bp}
1299: \put(7,177){\makebox(0,0){\small{$\xh^\EBP$}}}
1300: \put(170,0){\makebox(0,0){\small{$\ih$}}}
1301: \put(-8,-8){\makebox(0,0){\small{$0$}}}
1302: \put(153,-8){\makebox(0,0){\small{$1$}}}
1303: \put(-8,154){\makebox(0,0){\small{$1$}}}
1304: }
1305: \end{picture}
1306: \caption{EBP \exitentropy function $\left\{(\ih(\xl),\lnode(\xr(\xl)))\right\}_\xl$.}
1307: \label{fig:ebpexit}
1308: \vspace{2bp}
1309: %\end{floatingfigure}
1310: \end{figure}
1311:
1312: Surprisingly, we can apply the Generalized Area Theorem also to BP decoding if
1313: we consider the {\em Extended} BP \exitentropy (EBP) curve.
1314: %Let us now show that we can extend the AT
1315: %to BP decoding if we consider the {\em extended} BP \exitentropy curve.
1316: Fig.~\ref{fig:ebpexit} shows this
1317: EBP \exitentropy curve for the running example, i.e., for the \ddp $(\xl^2,\xr^5)$.
1318: We will see shortly that this
1319: EBP \exitentropy curve plays a central role in our investigation. First, let us give its formal
1320: definition.
1321:
1322: \bdefi
1323: Assume we are given a \ddp $(\ledge,\redge)$.
1324: The EBP \exitentropy curve, denote it by $\xh^{\EBP}$, is given in parametric form by
1325: \begin{align*}
1326: (\ih,\xh^{\EBP}) & = \left( \ih(\xl), \lnode(1-\redge(1-\xl)) \right),
1327: \end{align*}
1328: where $\ih(\xl) = \frac{\xl}{\ledge(1-\redge(1-\xl))}$ and $\xl \in [0, 1]$.
1329: \edefi
1330: %
1331: \btheo[Area Theorem for EBP Decoding] \label{theo:ebpareatheorem}
1332: Assume we are given a \ddp $(\ledge,\redge)$ of design rate $\drate$.
1333: Then the EBP \exitentropy curve satisfies
1334: \begin{align*}
1335: \int_{0}^1 \xh^{\EBP}(\xl) \text{d} \ih(\xl) = \drate.
1336: \end{align*}
1337: \etheo
1338: \bproof
1339: We will give two proofs of this fact.
1340: \par
1341: %\begin{floatingfigure}{85bp}
1342: \begin{figure}[hbt]
1343: \centering
1344: %\vspace{8bp}
1345: \setlength{\unitlength}{0.5bp}
1346: \begin{picture}(160,160) % size of small graph
1347: \put(20,0){\includegraphics[scale=0.5]{smallgraph}}
1348: \put(100,10){\makebox(0,0){\small{leaves}}}
1349: \put(100,150){\makebox(0,0){\small{root}}}
1350: \end{picture}
1351: \caption{Graph of a small tree code:
1352: computation tree of depth one for the regular (2,4) %($\ldegree=2$, $\rdegree=4$)
1353: LDPC ensemble.}
1354: \vspace{7bp}
1355: \label{fig:smallgraph}
1356: \end{figure}
1357: %\end{floatingfigure}
1358: $(i)$ The first proof applies only if $\ih(\xl)\leq1$ for $\xl\in(0,1]$. This in turn happens only if $\ledge'(0) > 0$, i.e., if the ensemble has a non-trivial
1359: stability condition.
1360: We use the (General) Area Theorem for transmission over binary
1361: erasure channels where we allow the parameter of the channel to vary as a function
1362: of the bit position.
1363: First, let us assume that the ensemble is $(\ldegree, \rdegree)$-regular. Consider a
1364: variable node and the corresponding computation tree of depth one as shown in Fig.~\ref{fig:smallgraph}.
1365: Let us further define two channel families. The first is the family $\{\text{BEC}(\xl) \}_{\xl=0}^{1}$.
1366: The second one is the family\footnote{Recall
1367: that $0 \leq \ih(\xl)\leq 1$ for all $\xl\in[0,1]$ by
1368: assumption.}
1369: $\{\text{BEC}(\ih(\xl)) \}_{\xl=0}^{1}$ where
1370: $\ih(\xl) \defas \frac{\xl}{\ledge(1-\redge(1-\xl))}$). The two families are parametrized
1371: by a common parameter $\xl$ which is the
1372: fixed-point of density evolution: they are smooth since $\ih(\xl)$ is differentiable with
1373: respect to $\xl$. Let us now assume that the bit associated to the
1374: root node is passed through a channel BEC$(\cp(\xl))$, while the ones
1375: associated to the leaf nodes are passed through a channel BEC$(\xl)$.
1376: We can apply the General Area Theorem: let $X=(X_1,\dots,X_{1+\ldegree\times(\rdegree-1)})$
1377: be the transmitted codeword chosen uniformly at random from the tree code and $Y(\xl)$ be the result of
1378: passing $X$ through the respective erasure channels parameterized by the common
1379: parameter $\xl$. The General Area Theorem states that
1380: $H(X \mid Y(\xl=1))-H(X \mid Y(\xl=0)) =H(X)$ is equal to the sum of the integrals
1381: of the individual \exitentropy curves, where the integral extends from $\xl=0$ to $\xl=1$.
1382: There are two types of individual \exitentropy curves, namely the one
1383: associated to the root node, call it $\xh_{\text{root}}(\xl)$
1384: and the $\ldegree (\rdegree-1)$ ones associated to the leaf nodes,
1385: call them $\xh_{\text{leaf}}(\xl)$. To summarize, the General Area Theorem
1386: states
1387: \begin{align*}
1388: H(X) & = \int_0^1\xh_{\text{root}}(\xl)\, \text{d}\cp(\xl)
1389: +\ldegree (\rdegree-1) \int_0^1\xh_{\text{leaf}}(\xl)\text{d}\xl.
1390: \end{align*}
1391: Note that $H(X) = 1+\ldegree (\rdegree-1)-\ldegree=1-\ldegree (\rdegree-2)$ since the computation tree
1392: contains $1+\ldegree(\rdegree-1)$ variable nodes and $\ldegree$ check nodes. Moreover, $\int_0^1\xh_{\text{leaf}}(\xl)\text{d}\xl=\int_0^1 1-\redge(1-\xl)\text{d}\xl=({\rdegree-1})/{\rdegree}$ since the message flowing from the root node to the check nodes is erased with probability $\xl$ (Recall that $\xl=\ih(\xl)\ledge(1-\redge(1-\xl))$, where $(\ledge(\xl),\redge(\xl))=(\xl^{\ldegree-1},\xl^{\rdegree-1})$. Moreover, observe that the result could also be obtained by applying the Area Theorem locally to the Single-Parity-Check code).
1393: Collecting these observations and solving for $\int_{0}^{1} \xh_{\text{root}}(\xl) d \cp(\xl)$, we get
1394: $$
1395: \int_{0}^{1} \xh_{\text{root}}(\xl) d \cp(\xl) = 1- \ldegree/\rdegree=\drate,
1396: $$
1397: as claimed since $\xh_{\text{root}}=\xh^\EBP$.
1398: %
1399: The irregular case follows in the same manner: we consider the ensemble of computation
1400: trees of depth one where the degree of the root note is chosen according to the
1401: node degree distribution $\lnode(\xl)$ and each edge emanating from this root node
1402: is connected to a check node whose degree is chosen according to the edge degree distribution
1403: $\redge(x)$. As before, leaf nodes experience the channel BEC$(\xl)$, whereas the root node
1404: experiences the channel BEC$(\cp(\xl))$. We apply the General Area Theorem to each such choice and
1405: average with the respective probabilities.
1406:
1407: $(ii)$ The second proof applies in all cases.
1408: % and requires calculus.\footnote{
1409: % Similar computations will be performed several times through this paper. They are
1410: % done in two steps reported as Lemma \ref{lemma:ibp1} and Lemma \ref{lemma:ibp2}
1411: % in Appendix \ref{sec:tricks}. This proof exemplifies this kind of computation. }
1412: Applying integration by parts twice we can write
1413: \begin{align*}
1414: \int_{0}^1 \xh^{\EBP}(\xl) \text{d} \ih(\xl)
1415: &= \xh^{\EBP}(\xl) \ih(\xl) \bigr|_{\xl=0}^{1} - \int_{0}^1 \frac{\text{d}\xh^{\EBP}(\xl)}{\text{d}\xl} \ih(\xl) \text{d} \xl\\
1416: &\overset{(a)}{=} 1- \lnode'(1) \int_{0}^1 x \redge'(1-x) \text{d} \xl\\
1417: &= 1- \frac{\Bigl(x \redge(1-x) \bigr|_{\xl=0}^{1} + \int_{0}^1 \redge(1-x) \text{d} \xl \Bigr)}{\int_0^1 \ledge(\xl) \text{d} \xl}\\
1418: &= 1- \lnode'(1)/\rnode'(1) = \drate,
1419: \end{align*}
1420: where $(a)$ follows since $\xh^{\EBP}(\xl)=\lnode'(1) \int_0^{1-\redge(1-\xl)}\ledge(\xl)\text{d}\xl$ and $\lnode'(1)=1/\int_0^1 \ledge$. Similar computations will be performed several
1421: times throughout this paper. In this respect it is handy to be able to refer to two
1422: basic facts related to this integration which are summarized as
1423: Lemma \ref{lemma:ibp1} and Lemma \ref{lemma:ibp2}
1424: in Appendix \ref{sec:tricks}. %This proof exemplifies this kind of computation.
1425: \eproof
1426: %
1427: %***************************************************************************
1428: %
1429: %\section{An Upper-Bound for the Maximum-Likelihood Threshold}
1430: \section{An Upper-Bound for the Maximum A Posteriori Threshold}
1431:
1432: \label{UpperSection}
1433:
1434: Assume that transmission takes places over BEC$(\ih)$. Given a \ddp
1435: $(\ledge,\redge)$, we trivially have the relations
1436: \begin{align}
1437: \label{equ:trivialrelations}
1438: \ih^\BP\leq \ih^\MAP \leq \min\{\ih^\text{Sh},\ih^\text{Stab}\},
1439: \end{align}
1440: where $\cp^\text{Sh}$ and $\cp^\text{Stab}$ denote, respectively,
1441: the Shannon and stability threshold.
1442: As we have discussed, it is straightforward to compute $\ih^\BP$ by means of
1443: DE and $\ih^\BP\leq \ih^\MAP $ follows from the sub-optimality
1444: of BP decoding.
1445: The inequality $\ih^\MAP\leq \ih^\text{Sh}=1 -\drate$ is a rephrasing of
1446: the Channel Coding Theorem.
1447: Finally $\ih^{\MAP}\le \ih^{\text{Stab}}=1/(\ledge'(0)\redge'(1))$
1448: can be proved through the following graph-theoretic argument.
1449: Assume, by contradiction that $\ih^{\MAP} > \ih^{\text{Stab}}$
1450: and let $\ih$ be such that $\ih^{\text{Stab}}<\ih <\ih^{\MAP}$.
1451: Notice that $\ih^{\text{Stab}}<\ih $ is equivalent to
1452: $\ih \ledge'(0)\redge'(1)>1$. Consider now the residual Tanner graph once the
1453: received variable nodes have been pruned, and focus on the subgraph of
1454: degree 2 variable nodes. Such a Tanner graph can be identified with an
1455: ordinary graph by mapping the check nodes to vertices and the variable nodes
1456: to edges. The average degree of such a graph is $\ih \ledge'(0)\redge'(1)>1$
1457: and therefore a finite fraction of its vertices belong to loops
1458: \cite{Bol01}. If a bit belongs to such a loop, it is not determined by the
1459: received message: in particular $\expectation[X_i|Y] = 1/2$.
1460: In fact, there exist a codeword such that $x_i=1$: just set $x_j=1$ if
1461: $j$ belongs to some fixed loop through $i$ and $0$ otherwise. Since there is
1462: a finite fraction of such vertices $h(\cp)>0$ (if the limit exist)
1463: and therefore $\cp>\cp^{\MAP}$. We reached a contradiction, therefore
1464: $\ih^{\MAP} \le \ih^{\text{Stab}}$ as claimed.
1465:
1466: While $\ih^\text{Stab}$ and $\ih^\text{Sh}$ are simple quantities,
1467: the threshold $\ih^\MAP$ is not as easy to compute.
1468: In this section we will prove an {\em upper-bound} on $\ih^\MAP$
1469: in terms of the (extended) BP \exitentropy curve. In the next sections,
1470: we will see that in fact this bound
1471: is tight for a large class of ensembles. The key to this
1472: bound is to associate the Area Theorem with the following intuitive inequality.
1473: \blemma \label{lemma:orderingviaphysicaldegradation} Consider a \ddp $(\ledge, \redge)$ and the associated EXIT functions $\xh^\BP$
1474: and $\xh^\MAP$. Then
1475: $\xh^\MAP\leq \xh^\BP$.
1476: \elemma
1477: \begin{proof}
1478: Note that Lemma \ref{lemma:orderingviaphysicaldegradation} expresses the natural
1479: statement that BP processing is in general suboptimal.
1480: For a given length $\n$, pick a code at random from $\ldpc(\ledge,\redge,\n)$.
1481: Call $\Phi^{\BP}_i$ the extrinsic BP
1482: estimate of bit $i$ and note that $\Phi^{\BP}_i=\Phi^{\BP}_i(Y_{\sim i})$,
1483: i.e., the extrinsic BP estimate is a well defined function of $Y_{\sim i}$.
1484: The Data Processing Theorem asserts that
1485: $H(X_i|Y_{\sim i})\leq H(X_i|\Phi^{\BP}_i(Y_{\sim i}))$. This is true for
1486: all codes in $\ldpc(\ledge,\redge,\n)$. Therefore taking first the
1487: average over the ensemble and second the limit
1488: when the blocklength $\n\to \infty$ (assuming the limit of the
1489: MAP EXIT function exists), we get $\xh^\MAP(\ih)\leq \xh^\BP(\ih)$.
1490: \end{proof}
1491:
1492:
1493: Because of Lemma \ref{lemma:orderingviaphysicaldegradation}, it is
1494: of
1495: course not surprising that the integral under $\xh^\BP$
1496: is larger or equal than the asymptotic rate of the code $r_{\text{as}}$
1497: as pointed out in Section \ref{sec:bpexitfunction}. In most of
1498: the cases encountered in practice, $\drate=r_{\text{as}}$, (see Section \ref{sec:Counting}),
1499: the area under the MAP EXIT curve is therefore $\drate$ and the area
1500: under the BP EXIT curve is strictly larger than $\drate$ if and only if
1501: the curve exhibits discontinuities (in the absence of discontinuities, the
1502: two curves coincide and the MAP/BP threshold is given by the stability condition).
1503:
1504: Example \ref{ex:stco} refines and illustrates this observation by showing that the
1505: BP and MAP threshold might be equal even if their respective \exitentropy functions
1506: are not pointwise equal.
1507: \bex\label{ex:stco}
1508: Consider the \ddp $(\ledge,\redge)=(0.4 x + 0.6 x^6,x^6)$ and the
1509: corresponding $\ldpc$ ensemble with design rate $\drate=0.5$. Using a weight enumerator
1510: function, see, e.g., Section \ref{sec:Counting}, one can show that
1511: $\drate=r_{\text{as}}=\int \xh^\MAP$. A quick look
1512: shows that the BP threshold is given by the stability condition, i.e.,
1513: it is $\ih^\BP\approx0.4167$ obtained for $\xl\approx\lxl^0=0$. When the
1514: parameter is $\uxl^0\approx0.04828$, i.e., at $\ih^1\approx0.4691$, a discontinuity
1515: of the BP \exitentropy curve appears and the edge erasure
1516: probability $\xl$ ``jumps'' to $\lxl^1\approx0.3309$. This situation is
1517: shown in Fig.~\ref{fig:epsilonwith1JandStabCond}. %Note
1518: %that the simple upper-bound found using the BP \exitentropy would be
1519: %%$\overline{\ih}^\BP \approx 0.4940$ obtained for $\xl \approx 0.4332$.
1520: \begin{figure}[htp]
1521: \centering
1522: \setlength{\unitlength}{1bp}
1523: \begin{picture}(240,140)
1524: \put(5,28)
1525: {
1526: \put(5,5){\includegraphics[width=100bp]{EPSwith1JandStabCond.ps}}
1527: \put(135,5){\includegraphics[width=100bp]{EXITwith1JandStabCond.ps}}
1528: % coordinates
1529: \put(0,0){\makebox(0,0){\small{$0$}}}
1530: \put(100,0){\makebox(0,0){\small{$1$}}}
1531: \put(0,100){\makebox(0,0){\small{$1$}}}
1532: \put(130,0){\makebox(0,0){\small{$0$}}}
1533: \put(230,0){\makebox(0,0){\small{$1$}}}
1534: \put(130,100){\makebox(0,0){\small{$1$}}}
1535: %% epsilon
1536: \put(5,-6){\makebox(0,0){\small{$\lxl^0$}}}
1537: %%\put(14,-6){\makebox(0,0){\small{$\uxl^0$}}}
1538: %\put(37,-6){\makebox(0,0){\small{$\lxl^1$}}}
1539: \put(-1,43){\makebox(0,0){\small{$\ih^\BP$}}}
1540: \put(-1,53){\makebox(0,0){\small{$\ih^1$}}}
1541: \put(60,80){\makebox(0,0){\small{$\ih(\xl)$}}}
1542: % EXIT
1543: %\put(176,-5){\makebox(0,0){\small{$\ih^\BP$}}}
1544: \put(185,17){\makebox(0,0){\small{$\ih^1$}}}
1545: \put(200,80){\makebox(0,0){\small{$\xh^\BP(\ih)$}}}
1546: }
1547: %\caption
1548: \put(60,7){\makebox(0,0){{\small{(a)}}}}
1549: \put(190,7){\makebox(0,0){{\small{(b)}}}}
1550: \end{picture}
1551: \caption{
1552: BP \exitentropy entropy curve with 1 discontinuity ($J$=1) for which the BP threshold $\ih^\BP=\ih^\MAP$ is given by the stability condition:
1553: (a) Channel entropy function $\xl\mapsto\ih(\xl)$ (b) BP \exitentropy function $\ih\mapsto\xh^\BP(\ih)$.
1554: %The function $\erasurepch(\erasurepex)$ has a first minimum at $\fpit = \erasurepex^{(1)}=0$ and second one at $\erasurepex^{(3)}=0.0482835$ for which the first discontinuity occurs.
1555: %This example is obtained for the \ddp $(\ledge,\redge)=(0.3x+0.3x^2+0.4x^{13},x^6)$
1556: %with design rate $\drate\approx0.48718$.
1557: %The BP threshold is $\ih^\BP\approx0.48437$ at $\xl^\BP\approx0.09904$.
1558: %This is also the first discontinuity, i.e., $\ih^1=\ih^\BP$, $\lxl^1=\xl^\BP$ and $\uxl^1\approx0.22156$.
1559: %The second discontinuity occurs for $\ih=\ih^2\approx0.51553$ at $\lxl^2\approx0.37016$ ($\uxl^1=1$).
1560: }
1561: \label{fig:epsilonwith1JandStabCond}
1562: \end{figure}
1563: Since the BP threshold is determined by the
1564: stability condition,
1565: as explained
1566: previously %in the beginning of the section,
1567: %Section
1568: %\ref{UpperSection},
1569: we have $\ih^\BP=\ih^\MAP$.
1570: This is true despite the fact that the integral
1571: under the BP \exitentropy is larger than $\drate=r_{\text{as}}$!
1572: \eex
1573:
1574:
1575:
1576:
1577:
1578:
1579: Recall that the Area Theorem asserts that
1580: $\int_0^1\xh^\MAP(\cp)\text{d}\cp=\drate_{\text{as}}$, where
1581: $\drate_{\text{as}}$ is the asymptotic rate of the ensemble defined in
1582: Theorem \ref{theo:asymptoticareatheorem}. By definition $\xh^\MAP(\cp)=0$ for
1583: $\cp \leq \cp^\MAP$. Therefore we have in fact
1584: $\int_{\cp^\MAP}^1\xh^\MAP(\cp)\text{d}\cp=\drate_{\text{as}}$.
1585: Now note that the BP decoder is in general suboptimal so that
1586: $\xh^\MAP(\cp) \leq \xh^\BP(\cp)$. Further, in general $\drate_{\text{as}} \geq \drate(\ledge, \redge)$.
1587: Combining these statements we see that
1588: if $\overline{\ih}^\MAP$ is a real number in $[\cp^\BP, 1]$ such that
1589: $\int_{\overline{\cp}^\MAP}^1 \xh^\BP(\cp) \text{d}\cp=\drate(\ledge, \redge)$ then
1590: $\int_{\overline{\cp}^\MAP}^1 \xh^\MAP(\cp) \text{d}\cp \leq \drate_{\text{as}}$.
1591: We conclude that for such a $\overline{\ih}^\MAP$,
1592: $\cp^\MAP \leq \overline{\cp}^\MAP$. Let us summarize a slightly strengthened version
1593: of this observation as a lemma.
1594: %
1595: \blemma[First Upper Bound on $\cp^\MAP$]
1596: \label{First}
1597: %
1598: Assume we are given a \ddp $(\ledge, \redge)$. Let $\xh^\BP(\cp)$
1599: denote the associated BP \exitentropy function and let
1600: $\overline{\ih}^\MAP$ be the unique real number in $[\cp^\BP, 1]$ such that
1601: $\int_{\overline{\cp}^\MAP}^1 \xh^\BP(\cp) \text{d}\cp=\drate(\ledge, \redge)$.
1602: Then $\cp^\MAP \leq \overline{\cp}^\MAP$. If in addition
1603: $\overline{\cp}^\MAP=\cp^\BP$ then $\cp^\MAP = \cp^\BP$,
1604: and in fact $\xh^\MAP(\cp)=\xh^\BP(\cp)$ for all $\cp \in [0, 1]$.
1605: %
1606: \elemma
1607: %
1608: \bproof
1609: We have already discussed the first part of the lemma. To see the second part,
1610: if $\overline{\cp}^\MAP=\cp^\BP$ then by (\ref{equ:trivialrelations})
1611: we have a lower and an upper bound that match and therefore we have equality.
1612: This can only happen if the two \exitentropy functions are in fact identical
1613: (and if $\drate_{\text{as}}=\drate(\ledge, \redge)$).
1614: \eproof
1615: \bex
1616: For the \ddp $(\ledge(x),\redge(x))=(x,x^{3})$,
1617: we obtain $\overline{\ih}^\MAP = 1/3 = \ih^\BP$.
1618: Therefore, for this case the MAP \exitentropy function is equal to the BP \exitentropy function
1619: and in particular both decoders have equal thresholds.
1620: \eex
1621: \bex
1622: For the \ddp $(\ledge(x),\redge(x))=(x^{2},x^{3})$,
1623: we obtain $\overline{\ih}^\MAP = \frac{102-7 \sqrt{21}}{108} \approx 0.647426$.
1624: Note that this \ddp has rate $1/4$ so that this upper bound on the
1625: threshold should be compared to the Shannon limit $3/4=0.75$.
1626: \eex
1627: \bex
1628: \label{exa:36upperbound}
1629: For the \ddp $(\ledge(x),\redge(x))=(x^2, x^5)$ of our running example, we get
1630: \centerline{
1631: $\overline{\ih}^\MAP = \frac{7 - {\sqrt{-1 - a + b}} - {\sqrt{-2 + a - b +
1632: \frac{4}{{\sqrt{-1 - a + b}}}}}}{ 6 {\left( -1 + {\left( - \frac{1}{6}
1633: + \frac{{\sqrt{-1 - a + b }}}{6} + \frac{{\sqrt{- 2 + a - b +
1634: \frac{4}{{\sqrt{-1-a+b}}}}}}{6} \right) }^5 \right) }^2},$}
1635: with $a \defas \frac{7\cdot 5^{\frac{2}{3}}}{{\left( 11 +
1636: 6\,{\sqrt{51}} \right) }^{\frac{1}{3}}} $ and
1637: $b \defas {\left( 55 + 30\,{\sqrt{51}} \right) }^{\frac{1}{3}}$.
1638: Numerically, $\overline{\ih}^\MAP \approx 0.4881508841915644$.
1639: The Shannon threshold for this ensemble is $0.5$.
1640: \eex
1641: For a \ddp which exhibits a single jump the computation of this upper bound
1642: is made somewhat easier by the following lemma. Note that by Fact \ref{fact:regular}
1643: this lemma is applicable to regular ensembles.
1644: \blemma \label{theo:exactmlthreshold}
1645: Assume we are given a \ddp $(\ledge,\redge)$.
1646: Define the polynomial $\xr(\xl)\defas1-\redge(1-\xl)$ and, for $\xl\in(0,1]$ the
1647: function $\ih(\xl)\defas\frac{\xl}{\ledge(\xr(\xl))}$. Assume that $\ih(\xl)$ is
1648: increasing over $[\xl^\BP,1]$. Let $\xl^*$ be the unique
1649: root of the polynomial
1650: $$
1651: P(\xl) \defas \lnode'(1)\xl(1-\xr(\xl)) -\frac{\lnode'(1)}{\Gamma'(1)}
1652: [1-\Gamma(1-\xl)]+\epsilon(\xl)\lnode(\xr(\xl))\, ,\label{eq:Pdef}
1653: $$
1654: in the interval $[\xl^\BP,1]$.
1655: Then $\overline{\ih}^\MAP = \ih(\xl^*)$.
1656: \elemma
1657: \bproof
1658: Recall that if $\ih(\xl)$ is increasing over $[\xl^\BP,1]$ then
1659: we have the parametric representation of $\xh^\BP(\cp)$ as given
1660: in (\ref{equ:bpexitcurve}). Using Lemmas \ref{lemma:ibp1} and
1661: \ref{lemma:ibp2} we can express the integral
1662: $\int_{\overline{\cp}^\MAP}^1 \xh^\BP(\cp) \text{d}\cp$
1663: as a function of $\overline{\cp}^\MAP$. More precisely, we
1664: parametrize $\overline{\cp}^\MAP$ by $\xl$ and express the integral
1665: as a function of $\xl$.
1666: Equating the result to $\drate(\ledge, \redge)=1-\lnode'(1)/\rnode'(1)$
1667: and solving for $\xl$ leads to the polynomial
1668: condition $P(\xl)=0$ stated above.
1669: \eproof
1670: \bex
1671: The following table compares the thresholds and bounds for various ensembles.
1672: Hereby
1673: $\ledge^{(1)}(x)=x$,
1674: $\ledge^{(2)}(x)=\frac{7 x^2 + 2 x^3 + 1 x^4}{10}$,
1675: $\ledge^{(3)}(x)=\frac{2857 x +3061.47 x^2+ 4081.53 x^9}{10000}$,
1676: $\ledge^{(4)}(x)=\frac{7.71429 x^2 + 2.28571 x^7}{10}$, and
1677: $\ledge^{(5)}(x)=\frac{9 x^2 + 1 x^7}{10}$.
1678: The threshold of the first ensemble is given by the stability condition. Its
1679: exact value is $7/28\approx0.1786$.
1680: \tablespace
1681: \small{
1682: \begin{tabular}{cccccc}
1683: %\hline
1684: $\ledge(x)$ & $\redge(x)$ & $\ih^\BP$ & $\overline{\ih}^\MAP$ & $\cp^\MAP$ & $\ih^\Sh$ \\ \hline
1685: $\ledge^{(1)}(x)$ & $\frac{2 x^5 + 3 x^6}{5} $ & 0.1786 & 0.1786 & 0.1786 & 0.3048 \\
1686: $\ledge^{(2)}(x)$ & $\frac{2 x^5 + 3 x^6}{5} $ & 0.4236 & 0.4948 & 0.4948 & 0.5024 \\
1687: $\ledge^{(3)}(x)$ & $x^6$ & 0.4804 & 0.4935 & 0.4935 & 0.5000 \\
1688: $\ledge^{(4)}(x)$ & $x^4$ & 0.5955 & 0.6979 & 0.6979 & 0.7000 \\
1689: $\ledge^{(5)}(x)$ & $x^7$ & 0.3440 & 0.3899 & 0.3899 & 0.4000 \\
1690: \end{tabular}
1691: }
1692: \tablespace
1693: \eex
1694: The polynomial $P(\xl)$ provides in fact a fundamental characterization
1695: of the MAP threshold and has some important properties.
1696: These are more conveniently stated in terms of a slightly more general concept.
1697: \bdefi\label{TrialDef}
1698: The {\em trial entropy} for the channel BEC$(\cp)$
1699: associated to the
1700: \ddp $(\ledge,\redge)$ is the bi-variate polynomial
1701: $$
1702: P_{\cp}(\xl,\xr) \defas \lnode'(1)\xl(1-\xr) -\frac{\lnode'(1)}{\rnode'(1)}
1703: [1-\rnode(1-\xl)]+\cp\lnode(\xr)\, .
1704: $$
1705: \edefi
1706: A few properties of the trial entropy are listed in the following.
1707: %
1708: \blemma\label{TrialLemma}
1709: %
1710: Let $(\ledge,\redge)$ be a \ddp and $P_{\cp}(\xl,\xr)$ the corresponding
1711: trial entropy. Consider furthermore the DE equations
1712: for the ensemble $\xl_{\iter+1} = \cp\ledge(\xr_{\iter})$,
1713: $\xr_{\iter+1} = 1-\redge(1-\xl_\iter)$, $\iter$ being the iteration number. Then
1714: (in what follows we always consider $\xl,\xr\in[0,1]$)
1715: %
1716: \begin{enumerate}
1717: %
1718: \item The fixed points of density evolution are stationary points of the trial
1719: entropy. Vice versa, any stationary point of the trial entropy
1720: is a fixed point of density evolution.\label{FixedPoints}
1721: %
1722: \item $P(\xl) = P_{\cp(\xl)}(\xl,\xr(\xl))$.\label{Polynomial}
1723: %
1724: \item $P(\xl = 1) = P_{\cp=1}(\xl=1,\xr=1)=\drate(\ledge, \redge)$.\label{PolRate}
1725: %
1726: \item \label{Integral}
1727: Let $a\defas (\cp_a=\cp(\xl_a),h^{\EBP}(\xl_a))$ and
1728: $b\equiv (\cp_b=\cp(\xl_b),h^{\EBP}(\xl_b))$ be two points
1729: on the EBP \exitentropy curve (with $\xl_{a/b}\in (0,1]$)
1730: and define $\xr_{a/b} = 1-\redge(1-\xl_{a/b})$. Then
1731: %
1732: \[
1733: \int_{a}^{b}h^\EBP(\cp(\xl))\, d\cp(\xl) = P_{\cp_{b}}(\xl_b,\xr_b)-
1734: P_{\cp_{a}}(\xl_a,\xr_a)\, .
1735: \]
1736: %
1737: \end{enumerate}
1738: %
1739: \elemma
1740: %
1741: \bproof
1742: (\ref{FixedPoints}) is proved by explicitly computing the partial derivatives
1743: of $P_{\cp}(\xl,\xr)$ with respect to $\xl$ and $\xr$:
1744: $\partial_{\xl} P_{\cp}(\xl,\xr) = \lnode'(1)[1-\xr-\redge(1-\xl)]$,
1745: $\partial_{\xr} P_{\cp}(\xl,\xr) = \lnode'(1)[-\xl+\cp\ledge(\xr)]$.
1746: Since $\lnode'(1)>0$, the stationarity conditions
1747: $\partial_{\xl} P_{\cp}(\xl,\xr) = 0$ and
1748: $\partial_{\xr} P_{\cp}(\xl,\xr) = 0$ are equivalent to the
1749: fixed point conditions for DE.
1750: (\ref{Polynomial}) and (\ref{PolRate}) are elementary algebra.
1751: In order to prove (\ref{Integral}), notice that we have
1752: $\partial_{\xl} P_{\cp}(\xl,\xr) = \partial_{\xr} P_{\cp}(\xl,\xr) = 0$
1753: at any point $(\xl,\xr(\xl),\cp(\xl))$ along the EBP \exitentropy curve.
1754: This follows from the fact that points on the EBP \exitentropy curve are
1755: fixed points of density evolution. Therefore
1756: %
1757: \begin{align*}
1758: %
1759: \frac{\de \phantom{\xl}}{\de \xl} P_{\cp(\xl)}(\xl,\xr(\xl)) & =
1760: \lnode(\xr(\xl))\, \frac{\de\cp}{\de \xl} (\xl) = h^\EBP(\cp(\xl))
1761: \, \frac{\de\cp}{\de \xl} (\xl)\, .
1762: %
1763: \end{align*}
1764: The thesis follows by integrating over $\xl$. Equivalently, we could
1765: have used again Lemmas \ref{lemma:ibp1} and \ref{lemma:ibp2}.
1766: % used, e.g., in Theorem \label{theo:ebpareatheorem}
1767: %or Lemma \label{theo:exactmlthreshold} and
1768: %recalled in Section \label{sec:tricks}.
1769: \eproof
1770:
1771: Unfortunately, the upper-bound stated in Lemma \ref{First} is not always
1772: tight. In particular, this can happen if the EBP \exitentropy curve exhibits
1773: multiple jumps (i.e., if $\cp(\xl)$ has more than one local maximum
1774: in the interval $(0,1]$). We will state a precise sufficient condition for tightness
1775: in the next section.
1776: An improved upper bound is obtained as follows.
1777: \btheo[Improved Upper-Bound on $\cp^\MAP$]\label{ImprovedUB}
1778: Assume we are given a \ddp $(\ledge, \redge)$. Let $\xh^\EBP(\cp)$
1779: denote the associated EBP \exitentropy function and let
1780: $(\overline{\ih}^\MAP=\cp(\xl^*),\xh^{\EBP}(\xl^*))$
1781: be a point on this curve. Assume that
1782: $\int_{\xl^*}^1 \xh^\EBP(\xl) \text{d}\cp(\xl)=\drate(\ledge, \redge)$
1783: and that
1784: there exist
1785: no $\xl'\in (\xl^*, 1]$ such that $\cp(\xl') = \cp(\xl^*)$.
1786: Then $\cp^\MAP \leq \overline{\cp}^\MAP$.
1787: \etheo
1788: The proof of this theorem will be given in Section \ref{MaxwellSection}
1789: using the so-called Maxwell construction. Notice that in general there can be more than one
1790: value of $\epsilon$ satisfying the theorem hypotheses. We shall always
1791: use the symbol $\overline{\cp}^\MAP$ to refer to the smallest such value. On the other
1792: hand, it is a consequence of the proof of theorem that there always exists
1793: at least one such value.
1794:
1795: As before, the following lemma simplifies the computation of the
1796: upper bound by stating the following more explicit characterization.
1797: \blemma
1798: \label{lem:exactmlthreshold2}
1799: Consider a \ddp $(\ledge,\redge)$.
1800: Let $\xl^*\in(0,1]$ be a root of the polynomial
1801: $P(\xl)$ defined in (\ref{eq:Pdef}), such that there exist
1802: no $\xl'\in (\xl^*, 1]$ with $\cp(\xl') = \cp(\xl^*)$.
1803: Then $\ih^\MAP \le \ih(\xl^*)$, and $\overline{\epsilon}^\MAP$
1804: is the smallest among such upper bounds.
1805: \elemma
1806: %
1807: \begin{proof}
1808: Let $\xl^*$ be defined as in the statement. Then, by Lemma
1809: \ref{TrialLemma}, points (\ref{Polynomial}), (\ref{PolRate}) and
1810: (\ref{Integral}):
1811: \[
1812: \int_{\xl^*}^1h^\EBP(\xl)\,\de\cp(\xl) = P(1)-P(\xl^*) = \drate(\ledge, \redge)-P(\xl^*)\, .
1813: \]
1814: Therefore, $\int_{\xl^*}^1h^\EBP(\xl)\,\de\cp(\xl) = \drate(\ledge, \redge)$ if and
1815: only if $P(\xl^*) = 0$.
1816: \end{proof}
1817:
1818: For a large family of \ddps the upper bound stated
1819: in Theorem \ref{ImprovedUB}
1820: is indeed tight. Nevertheless, it is possible to construct
1821: examples where we can not evaluate the bound at all roots
1822: $\xl^*$ of $P(\xl)$ since for some of those roots there exists a point
1823: $\xl'\in (\xl^*, 1]$ with $\cp(\xl') = \cp(\xl^*)$.
1824: In these cases we expect the bound not to be tight.
1825: Indeed, we conjecture that the extra condition on the roots
1826: of $P(\xl)$ are not necessary and that the MAP threshold is
1827: in general given by the following statement.
1828: %We expect even the refined upper bound \ref{ImprovedUB} not to
1829: %be tight for some (very) special degree distributions pairs.
1830: %However, there is a natural conjecture for the MAP threshold in the most
1831: %general ensemble.
1832: %
1833: \bconj
1834: \label{conj:mainconjecture}
1835: Consider a degree distribution pair $(\ledge,\redge)$ and the associated
1836: polynomial $P(\xl)$ defined as in (\ref{eq:Pdef}).
1837: Let $\cX\subset (0,1]$ be the set of positive roots of
1838: $P(\xl)$ in the interval $(0,1]$ (since $P(\xl)$ is a polynomial,
1839: $\cX$ is finite). Equivalently, $\cX$ is the set of
1840: $\xl_*\in (0,1]$ such that $\int_{\xl^*}^1h^\EBP(\xl)\,\de\cp(\xl) = \drate(\ledge, \redge)$.
1841: Then $\cp^{\MAP}=\min\{\cp(\xl^*);\, \xl\in\cX\}$.
1842: \econj
1843:
1844: %
1845: %*****************************************************************************
1846: %
1847: \section{Counting Argument}
1848: \label{sec:Counting}
1849:
1850: We will now describe a counting argument which yields
1851: an alternative proof of Lemma \ref{First}. More interestingly,
1852: the argument can be strengthened to obtain an easy-to-evaluate
1853: sufficient condition for tightness of the upper-bound.
1854:
1855: The basic idea is quite simple. Recall that we define the
1856: MAP threshold as the maximum of all channel parameters for which the normalized
1857: conditional entropy converges to zero as the block length tends to infinity.
1858: For the binary erasure channel, the conditional entropy is equal to the logarithm of the number of codewords
1859: which are compatible with the received word.
1860: Therefore, a first naive way of upper bounding the MAP threshold consists in lower bounding
1861: the expected number of codewords in the residual graph, after
1862: eliminating the received variables. If, for a given channel parameter, this
1863: lower bound is exponential with a strictly positive exponent,
1864: then the corresponding conditional entropy is strictly positive and
1865: we are operating above the threshold. It turns out that a much better
1866: result is obtained by considering the residual graph after iterative decoding
1867: has been applied. In fact, this simple modification allows one to obtain matching upper
1868: and lower bounds in a large number of cases.
1869:
1870: Let $\graph$ be chosen uniformly at random from the ensemble characterized by
1871: $\nddp \defas (\lnode,\rnode)$.
1872: Assume further that transmission takes place over BEC$(\cp)$ and
1873: that a BP decoder is applied to the received sequence.
1874: Denote by $\graph(\ih)$
1875: the residual graph after decoding has halted, and by
1876: $\nddp_{\graph(\cp)} = (\lnode_{\graph(\cp)},\rnode_{\graph(\cp)})$
1877: its degree profile (i.e., the fraction of nodes of any given degree).
1878: We adopt here the convention of normalizing the \ddp of $\graph(\cp)$
1879: with respect to the number of variable nodes and check nodes in the
1880: {\em original} graph. Therefore, $\lnode_{\graph(\cp)}(1)\le 1$
1881: is the number of variable nodes in $\graph(\cp)$
1882: divided by $\n$. Analogously, $\rnode_{\graph(\cp)}(1)\le 1$ is the number
1883: of check nodes in $\graph(\cp)$ divided by $n\lnode'(1)/\rnode'(1)$.
1884:
1885: It is shown in \cite{LMSS01b} that, conditioned on the degree profile
1886: of the residual graph, $\graph(\cp)$ is uniformly distributed.
1887: The \ddp $\nddp_{\graph(\cp)}$ itself is of course a random quantity because
1888: of the channel randomness.
1889: However, it is is sharply concentrated around its expected value. For
1890: increasing blocklengths this expected value converges to
1891: $\nddp_{\cp}=(\lnode_{\cp},\rnode_{\cp})$, which is given by\footnote{The standard \ddp from
1892: the node
1893: perspective of the residual graph when transmission takes place over BEC($\ih$)
1894: is then simply given by $\left(\frac{\lnode_{\cp}(\xl)}{\lnode_{\cp}(1)},\frac{\rnode_{\cp}(\xl)}{\rnode_{\cp}(1)}\right)$. }
1895: %
1896: \begin{align}
1897: %
1898: \lnode_{\cp}(\zl) & \defas \cp\lnode(\zl \xr)\, ,
1899: \label{LambdaRes}\\
1900: \Gamma_{\cp}(\zl) &\defas \Gamma(1-\xl+\zl\xl)-\Gamma(1-\xl)-
1901: \zl\xl\Gamma'(1-\xl)\, .
1902: \label{GammaRes}
1903: %
1904: \end{align}
1905: %
1906: Here, $\xl$ and $\xr$ denote the fraction of erased messages at the fixed point
1907: of the BP decoder. More precisely, $\xl\in [0,1]$
1908: is the largest solution
1909: of $\xl=\epsilon\ledge(1-\redge(1-\xl))$ and $\xr = 1-\redge(1-\xl)$.
1910: The precise concentration statement follows.
1911: %
1912: \blemma\label{ConcentrationDegreeLemma}
1913: %
1914: Let $\ih\in (0,1]$ be a continuity point of
1915: $\xl(\ih)$ (we shall call such an $\ih$ {\em non-exceptional}).
1916: Then, for any $\xi > 0$,
1917: %
1918: \begin{align}
1919: %
1920: \lim_{\n\to\infty}\Pr\{d(\nddp_{\graph(\ih)},\nddp_\ih)\ge \xi\} &=0\, .
1921: %
1922: \end{align}
1923: %
1924: Here, $d(\cdot,\cdot)$ denotes the $L_1$ distance
1925: %
1926: \begin{align}
1927: %
1928: d(\nddp,\tilde{\nddp}) & \defas \sum_{\ldegree}
1929: |\lnode_{\ldegree}-\tilde{\lnode}_{\ldegree}|+\sum_{\rdegree}
1930: |\rnode_{\rdegree}-\tilde{\rnode}_{\rdegree}|\, .
1931: %
1932: \end{align}
1933: %
1934: %
1935: \elemma
1936: %
1937:
1938: The proof is deferred to Appendix \ref{ProofCounting}.
1939:
1940: Under the zero-codeword assumption, the set of codewords compatible with
1941: the received bits coincides with the set of codewords of the residual graph.
1942: Their expected number can be computed through standard combinatorial tools.
1943: The key idea here is that,
1944: under suitable conditions on the \ddp (of the residual graph),
1945: the actual rate of codes from the (residual) ensemble is close to the design rate.
1946: We state here a slightly strengthened version of this result from \cite{DMU04}.
1947: \blemma
1948: \label{WELemma}
1949: Let $\graph$ be chosen uniformly at random from the ensemble
1950: LDPC$(\n,\nddp) = $LDPC$(\n,\lnode,\rnode)$, let $\rate_\graph$ be its rate and
1951: $\drate \defas 1-\lnode'(1)/\rnode'(1)$ be the design rate.
1952: Consider the function $\Psi_{\nddp}(u)$,
1953: %
1954: \begin{align}
1955: %
1956: \Psi_{\nddp}(u) =& -\lnode'(1)\log_2\left[\frac{(1+uv)}{(1+u)(1+v)}\right]
1957: \nonumber \\&+\sum_\ldegree\lnode_\ldegree\log_2\left[
1958: \frac{1+u^\ldegree}{2 (1+u)^\ldegree}\right]
1959: \nonumber\\
1960: &+\frac{\lnode'(1)}{\Gamma'(1)}\sum_{\rdegree}\Gamma_\rdegree
1961: \log_2\left[1+\left(\frac{1-v}{1+v}\right)^\rdegree\right]\, ,\label{Psidef} \\
1962: v = & \left(\sum_{\ldegree}\frac{\ledge_\ldegree}{1+u^\ldegree}
1963: \right)^{-1}\left(\sum_{\ldegree}\frac{\ledge_{\ldegree}u^{\ldegree-1}}
1964: {1+u^\ldegree}\right)\, .\label{xtoy}
1965: %
1966: \end{align}
1967: %
1968: Assume that $\Psi_{\nddp}(u)$ takes on its global maximum
1969: in the range $u\in[0,\infty)$ at $u=1$. Then there exists $B>0$ such that,
1970: for any $\xi>0$, and $\n>\n_0(\xi,\Xi)$,
1971: %
1972: \begin{align*}
1973: %
1974: \Pr\{|\rate_\graph- \drate(\lnode, \rnode)|>\xi\}&\le e^{-B n\xi}\, .
1975: %
1976: \end{align*}
1977: %
1978: Moreover, there exist $C>0$ such that, for $n>n_0(\xi,\Xi)$,
1979: %
1980: \begin{align*}
1981: %
1982: \expectation[ |\rate_\graph- \drate(\lnode, \rnode)|] &\le C\frac{\log n}{n}\, .
1983: %
1984: \end{align*}
1985: %
1986: %
1987: \elemma
1988: %
1989: \bproof
1990: The idea of the proof is the following. For any parity-check ensemble
1991: we have $\rate_{\graph}\ge \drate(\lnode, \rnode)$. If it is true
1992: that the expected value of the rate (more precisely, the logarithm of the
1993: expected number of codewords divided by the length) is close
1994: to the design rate, then we can use the Markov inequality to
1995: show that most codes have rate close to the design rate.
1996:
1997:
1998: Let us start by computing the exponent of the expected number
1999: of codewords. We know from
2000: \cite{Gal62,MB04,LS02,LS03,DRU01,DRU04,Di04,DMU04,SeF96,FlS94}
2001: that the expected number
2002: of codewords involving $E$ edges is given by
2003: \begin{align*}
2004: \expectation[N_{\graph}(E)] & =
2005: \frac{
2006: \text{coef}
2007: \left\{
2008: \prod_\ldegree (1+u^\ldegree)^{n \lnode_\ldegree}
2009: \prod_\rdegree q_\rdegree(v)^{n \frac{\lnode'(1)}{\rnode'(1)} \rnode_\rdegree}, u^E v^E \right\} }{
2010: \binom{n \lnode'(1)}{E}},
2011: \end{align*}
2012: where $q_\rdegree(v) = ((1+v)^\rdegree+(1-v)^\rdegree)/2$. Let $n$ tend to infinity and define $e=E/(n \lnode'(1)$.
2013: From standard arguments presented in the cited papers it is known that,
2014: for a fixed $e$, the exponent
2015: $\lim_{n \rightarrow \infty} \frac{1}{n}\log_2\bigl(\expectation[
2016: N_{\graph}(e n \lnode'(1))]\bigr)$
2017: is given by the infimum with respect to $u, v > 0$ of
2018: \begin{multline}
2019: \sum_{\ldegree}\lnode_{\ldegree}\log_2(1+u^{\ldegree})-\lnode'(1)
2020: e\log_2 u
2021: +\frac{\lnode'(1)}{\rnode'(1)}\sum_{\rdegree}
2022: \Gamma_{\rdegree}\log_2 q_{\rdegree}(v) \\
2023: -\lnode'(1) e\log_2 v-\lnode'(1)h(e).
2024: \label{equ:exponent}
2025: \end{multline}
2026: We want to determine the exponent corresponding to the expected number
2027: of codewords, i.e.,
2028: $\lim_{n \rightarrow \infty} \frac{1}{n}\log_2\bigl(\expectation[
2029: N_{\graph}]\bigr)$, where $N_{\graph}=\sum_{E} N_{\graph}(E)$.
2030: Since there is only a polynomial number of ``types''
2031: (numbers $E$) this exponent is
2032: equal to the supremum of (\ref{equ:exponent}) over all $0 \leq e \leq 1$.
2033: In summary, the sought after exponent is given by a stationary point of the function stated in
2034: (\ref{equ:exponent}) with respect to
2035: $u$, $v$ and $e$.
2036:
2037: Take the derivative with respect to $e$.
2038: This gives $e=uv/(1+uv)$. If we substitute this expression for $e$
2039: into (\ref{equ:exponent}), subtract the design rate
2040: $\drate(\lnode, \rnode)$,
2041: and rearrange the terms somewhat we get (\ref{Psidef}).
2042: Next, if we take the derivative with respect to
2043: $u$ and solve for $v$ we get get (\ref{xtoy}).
2044: In summary, $\Psi_{\nddp}(u)$ is a function so that
2045: %
2046: \begin{align*}
2047: %
2048: \log_2 \expectation [N_{\graph}] &=
2049: n\{\drate(\lnode, \rnode)+\sup_{u\in[0,\infty)}\Psi_{\nddp}(u) +\omega_n\}\, ,
2050: %
2051: \end{align*}
2052: %
2053: where $\omega_n = o(1)$. In particular, by explicit computation we see that
2054: $\Psi_{\nddp}(u=1) = 0$. A closer look shows that
2055: $u=1$ corresponds to the exponent of codewords of weight $n/2$.
2056: Therefore, the condition that the global maximum
2057: of $\Psi_{\nddp}(u)$ is achieved at $u=1$ is equivalent to the condition
2058: that the expected weight enumerator is dominated by codewords of weight
2059: (close to) $n/2$. Therefore,
2060: %
2061: \begin{align*}
2062: %
2063: \Pr\{\rate_{\graph}\ge \drate(\lnode, \rnode)+\xi\} &=
2064: \Pr\left\{N_{\graph}\ge 2^{n(\xi-\omega_n)} \expectation[N_{\graph}]
2065: \right\} \\
2066: & \le e^{-Bn\xi}\, ,
2067: %
2068: \end{align*}
2069: %
2070: where the step follows from the Markov inequality if
2071: $B=(\log 2)/2$ and $\omega_n\le\xi/2$ for any $n\ge n_0$.
2072:
2073: Finally, we observe that, since $\rate_{\graph}\le 1$
2074: %
2075: \begin{align*}
2076: %
2077: \expectation [|\rate_\graph- \drate(\lnode, \rnode)|] & \le \xi + e^{-Bn\xi}\, ,
2078: %
2079: \end{align*}
2080: %
2081: and the second claim follows by choosing $\xi = \log n/Bn$.
2082: %
2083: \eproof
2084: %
2085:
2086: We would like to apply this result to the residual graph $\graph(\cp)$.
2087: Since the degree profile of $\graph(\cp)$ is a random variable, we need
2088: a preliminary observation on the ``robustness'' of the hypotheses
2089: in the Lemma \ref{WELemma}.
2090: %
2091: \blemma\label{PsiProp}
2092: %
2093: Let $\Psi_{\nddp}(\cdot)$ be defined as in Lemma \ref{WELemma}.
2094: Then $\Psi_{\nddp}(u)$ achieves its maximum over $u\in[0,+\infty)$
2095: in $[0,1]$.
2096:
2097: Moreover,
2098: there exists a constant $A>0$ such that, for any two
2099: degree distribution pairs
2100: $\nddp =(\lnode,\rnode)$ and
2101: $\tilde{\nddp}=(\tilde{\lnode},\tilde{\rnode})$, and
2102: any $u\in [0,1]$,
2103: %
2104: \begin{align}
2105: %
2106: |\Psi_{\nddp}(u)-\Psi_{\tilde{\nddp}}(u)| &
2107: \le A\, d(\nddp,\tilde{\nddp})\, (1-u)^2\, .
2108: \label{PsiLip}
2109: %
2110: \end{align}
2111: %
2112: \elemma
2113: %
2114: For the proof we refer to Appendix \ref{ProofCounting}.
2115:
2116: We turn now to the main result of this section.
2117: %
2118: \btheo\label{CountingTheorem}
2119: Let $\graph$ be a code picked uniformly at random from the ensemble LDPC$(\n,\lnode,\Gamma)$
2120: and let $H_{\graph}(X|Y)$ be the conditional entropy of the transmitted
2121: message when the code is used for communicating over BEC$(\cp)$.
2122: Denote by $P_{\epsilon}(\xl,\xr)$ the corresponding trial entropy.
2123: Let $\nddp_{\cp}=(\lnode_{\cp},\Gamma_{\cp})$ be the typical degree
2124: distribution pair of the residual graph, see
2125: Eqs.~(\ref{LambdaRes}), (\ref{GammaRes}), and $\Psi_{\nddp_{\cp}}(x)$
2126: be defined as in Lemma \ref{WELemma}, Eq. (\ref{Psidef}).
2127:
2128: Assume that $\Psi_{\nddp_{\epsilon}}(u)$ achieves its global maximum as
2129: a function of $u\in[0,\infty)$ at $u = 1$, with
2130: $\Psi''_{\nddp_{\epsilon}}(1)<0$, and that $\epsilon$ is non-exceptional.
2131: Then
2132: %
2133: \begin{align}
2134: \label{equ:conditonalentropy}
2135: %
2136: \lim_{n\to\infty}\frac{1}{n}\expectation [H_{\graph}(X|Y)] & = P_{\epsilon}(\xl,\xr)
2137: \, ,
2138: %
2139: \end{align}
2140: %
2141: where $\xl\in [0,1]$ is the largest solution
2142: of $\xl=\epsilon\ledge(1-\redge(1-\xl))$ and $\xr = 1-\redge(1-\xl)$.
2143: \etheo
2144:
2145: \bproof
2146: As above, we denote by $\graph(\epsilon)$ the residual graph after
2147: BP decoding and by $\rate_{\graph(\epsilon)}$ its rate normalized
2148: to the original blocklength $n$.
2149: Notice that $H_{\graph}(X|Y) = n\rate_{\graph(\epsilon)}$: iterative decoding
2150: does not exclude any codeword compatible with the received bits.
2151: Furthermore, the design rate (always normalized to $n$) for the
2152: \ddp of the residual graph is
2153: %
2154: \begin{align*}
2155: %
2156: \drate(\nddp_{\graph(\cp)}) &= \lnode_{\graph(\cp)}(1)-
2157: \frac{\lnode'(1)}{\Gamma'(1)}\, \Gamma_{\graph(\cp)}(1)\, .
2158: %
2159: \end{align*}
2160: %
2161: We further introduce the notation $r_{\cp}$ for the design rate of the typical
2162: \ddp of the residual graph. Using Eqs.~(\ref{LambdaRes}) and
2163: (\ref{GammaRes}), we can find
2164: %
2165: \begin{align*}
2166: %
2167: \drate_{\cp} &= \lnode'(1)\redge(1-\xl)\xl-\frac{\lnode'(1)}{\Gamma'(1)}
2168: [1-\Gamma(1-\xl)]+\cp\lnode(\xr) \\
2169: & = P_{\cp}(\xl,\xr),
2170: %
2171: \end{align*}
2172: %
2173: where the last step follows from the fixed-point condition
2174: $\xr=1-\redge(1-\xl)$.
2175:
2176: Since by assumption $\Psi_{\nddp_{\cp}}(u)$ achieves its global maximum at
2177: $u = 1$, with $\Psi''_{\nddp_{\cp}}(1)<0$, and $\Psi_{\nddp_{\cp}}(1)=0$,
2178: there exists a positive constant $\delta$ such that
2179: $\Psi_{\nddp_{\cp}}(u)\le -\delta (1-u)^2$ for any $u\in[0,1]$.
2180: As a consequence of Lemma \ref{PsiProp}, there exist a $\xi>0$
2181: such that, for any \ddp $\nddp$, with $d(\nddp,\nddp_{\cp})\le \xi$,
2182: $\Psi_{\nddp}(u)\le -\delta (1-u)^2/2$ for $u\in[0,1]$.
2183:
2184: Let $\pr_{\cp}(\tilde{\nddp})$ be the probability that
2185: the degree distribution pair of the residual graph
2186: $\graph(\cp)$ is $\tilde{\nddp} = (\tilde{\lnode},\tilde{\Gamma})$.
2187: Denote by $\tilde{\expectation}$ expectation with respect to
2188: a uniformly random code in the $(\tilde{n},\tilde{\lnode},\tilde{\Gamma})$ ensemble (here $\tilde{n}\defas n\tilde{\lnode}(1)$).
2189: Denote by $\cN(\xi)$ the set of \ddps $\tilde{\nddp}$, such that
2190: $d(\tilde{\nddp},\nddp_{\cp})\le \xi$.
2191: The above remarks imply that we can apply Lemma \ref{WELemma} to any
2192: ensemble in $\cN(\xi)$.
2193: Then
2194: %
2195: \begin{align*}
2196: %
2197: \frac{1}{n}\expectation [H_{\graph}(X|Y)] & = \sum_{\tilde{\nddp}}
2198: \pr_{\cp}(\tilde{\nddp})\,
2199: \tilde{\expectation}[ \rate_{\graph(\epsilon)}] \\
2200: & = \!\!\!\sum_{\tilde{\nddp}\in {\cal N}(\xi)}\!\!\!
2201: \pr_{\epsilon}(\tilde{\nddp})\,
2202: \tilde{\expectation}[ \rate_{\graph(\epsilon)} ] + \omega(n,\xi).
2203: %
2204: \end{align*}
2205: %
2206: The remainder can be estimated by noticing that
2207: $\rate_{\graph(\epsilon)}\le 1$ while the probability of
2208: $\tilde{\nddp}\not\in {\cal N}(\epsilon)$ is bounded by Lemma
2209: \ref{ConcentrationDegreeLemma}. Therefore
2210: %
2211: \begin{align*}
2212: %
2213: \lim_{n\to\infty}\omega(n,\xi) & = 0\, .
2214: %
2215: \end{align*}
2216: %
2217:
2218: Now we can apply Lemma \ref{WELemma} to get
2219: %
2220: \begin{align*}
2221: %
2222: \left|\frac{1}{n}\expectation [ H_{\graph}(X|Y)] -
2223: \drate_{\cp}\right| \le &~~ \!\!\!\!\!\!
2224: \sum_{\tilde{\nddp}\in {\cal N}(\xi)}\!\!\!\!
2225: \pr_{\epsilon}(\tilde{\nddp})\, |\tilde{\expectation}[ \rate_{\graph(\epsilon)}]-
2226: \drate(\tilde{\nddp})| \\
2227: &~+\!\!\!\!\!\!\sum_{\tilde{\nddp}\in {\cal N}(\xi)}\!\!\!\!
2228: \pr_{\epsilon}(\tilde{\nddp})\, |\drate(\tilde{\nddp})-\drate_{\cp}| +
2229: \omega(n,\xi) \\
2230: \le&~ \!\!\!\!\!\!\sum_{\tilde{\nddp}\in {\cal N}(\xi)}\!\!\!\!
2231: \pr_{\epsilon}(\tilde{\nddp})\, |\drate(\tilde{\nddp})-\drate_{\cp}| +
2232: \omega'(n,\xi)\, ,
2233: %
2234: \end{align*}
2235: %
2236: where $\omega'(n,\xi) = \omega(n,\xi)+C\log n/n$.
2237: Notice that there exist $B>0$ such that for any pair $\nddp_1$, $\nddp_2$
2238: %
2239: \begin{align*}
2240: %
2241: |\drate(\nddp_1)-\drate(\nddp_2)| & \le
2242: B\, d(\nddp_1;\nddp_2)\, .
2243: %
2244: \end{align*}
2245: %
2246: Therefore,
2247: %
2248: \begin{align*}
2249: %
2250: \lim_{n\to\infty}\left|\frac{1}{n}\expectation [H_{\graph}(X|Y)] -
2251: \drate_{\cp}\right|&\le
2252: B \xi\, .
2253: %
2254: \end{align*}
2255: %
2256: The claim follows by noticing that $\xi$ can be chosen arbitrarily small.
2257: %
2258: \eproof
2259:
2260: Theorem \ref{CountingTheorem} allows to compute the exact
2261: MAP threshold whenever the required conditions are verified.
2262: An explicit characterization is given below.
2263: %
2264: \bcor
2265: %
2266: Consider transmission over BEC$(\cp)$ using
2267: elements picked uniformly at random from the ensemble $(\lnode,\Gamma)$. Let $\xl^*, \xr^*>0$
2268: be the DE fixed-point achieved by the BP
2269: decoder at a non-exceptional
2270: erasure probability $\cp^*$ (i.e., $\xl^*\in (0,1]$ is the largest
2271: solution of $\xl^* = \cp^*\ledge(1-\redge(1-\xl^*))$).
2272: Assume that $P_{\epsilon^*}(\xl^*,\xr^*)=0$ and that
2273: $\Psi_{\nddp_{\epsilon^*}}(u)\le 0$ for $u\in[0,+\infty)$ together with
2274: $\Psi''_{\nddp_{\epsilon^*}}(1)<0$. Let $\cW\subseteq[0,+\infty)$ be
2275: the set of points $u\neq 1$ such that $\Psi_{\nddp_{\epsilon^*}}(u)= 0$.
2276: If, for any $u\in\cW$, $\partial_{\cp}\Psi_{\nddp_{\epsilon^*}}(u)<
2277: \partial_{\cp}\Psi_{\nddp_{\epsilon^*}}(1)$, then $\cp^\MAP = \cp^*$.
2278: %
2279: \ecor
2280: %
2281:
2282: \bproof
2283: We claim that there exist a $\delta>0$ such that
2284: the hypothesis of Theorem \ref{CountingTheorem} are verified for any
2285: $\cp\in (\cp^*,\cp^*+\delta)$. Before proving this claim,
2286: let us show that it implies the thesis. Consider any $\cp\in (\cp^*,\cp^*+\delta)$
2287: and let $\xl,\xr$ be the corresponding density evolution fixed point.
2288: Then
2289: %
2290: \begin{align*}
2291: %
2292: \lim_{n\to\infty}\frac{1}{n}\expectation[H(X|Y)] &=
2293: P_{\cp}(\xl(\cp),\xr(\cp))\;\;\;\;\;\;\; \forall
2294: \cp\in (\cp^*,\cp^*+\delta)\, .
2295: %
2296: \end{align*}
2297: %
2298: Moreover $P_{\cp^*}(\xl(\cp^*),\xr(\cp^*))=0$ by hypothesis
2299: and
2300: %
2301: \begin{align*}
2302: %
2303: \frac{\de\phantom{\cp}}{\de\cp}P_{\cp}(\xl(\cp),\xr(\cp)) & = \lnode(\xr(\cp))>0
2304: \, .
2305: %
2306: \end{align*}
2307: %
2308: Therefore $P_{\cp}(\xl(\cp),\xr(\cp))>0$ for any $\cp>\cp^*$. This implies
2309: $\cp^\MAP\le\cp^*$. On the other hand $\expectation[H(X|Y)]$ is strictly
2310: increasing with $\cp$. This implies
2311: %
2312: \begin{align*}
2313: %
2314: \lim_{n\to\infty}\frac{1}{n}\expectation[H(X|Y)] &= 0,
2315: \;\;\;\;\;\;\; \forall \cp\in [0,\cp^*] ,
2316: %
2317: \end{align*}
2318: %
2319: which in turn implies $\cp^\MAP\ge \cp^*$ and, therefore, $\cp^\MAP = \cp^*$.
2320:
2321: Let us now prove the claim. By assumption $\cp^*$ is non-exceptional
2322: and therefore the residual \ddp $\nddp_{\cp}$ is continuous at
2323: $\cp^*$. This implies, via Lemma \ref{PsiProp}
2324: that, for any $\xi>0$, there exist $\delta$ such that
2325: for $\cp\in[\cp^*,\cp^*+\delta)$ and any $u\in[0,1]$,
2326: %
2327: \begin{align*}
2328: %
2329: |\Psi_{\nddp_\cp}(u)-\Psi_{\nddp_{\cp^*}}(u)| & \le \xi (1-u)^2\, .
2330: %
2331: \end{align*}
2332: %
2333: Together with $\Psi''_{\nddp_\cp}(1)<0$, this implies that, if
2334: $\delta$ is small enough, $u=1$ is a local maximum of $ \Psi_{\nddp_\cp}(u)$.
2335: It follows from the hypotheses on $\partial_{\cp}\Psi_{\nddp_{\cp^*}}(u)$,
2336: $u\in\cW$, that it is also a global maximum.
2337: \eproof
2338: %
2339:
2340: The conditions in the above corollary are relatively easy to verify.
2341: Let us demonstrate this by means of two examples.
2342: \bex[Ensemble LDPC($x^2,x^5$)] Consider the
2343: $(3,6)$-regular LDPC ensemble. For convenience of
2344: the reader its EBP EXIT curve is
2345: repeated in Fig.~\ref{fig:36ebpexit}.
2346: %\begin{floatingfigure}{115bp}
2347: \begin{figure}
2348: %\vspace{10bp}
2349: \centering
2350: \setlength{\unitlength}{0.5bp}
2351: \begin{picture}(210,230)
2352: \put(19,10)
2353: {
2354: \put(0,0){\includegraphics[scale=0.62]{EXIT36WD}}
2355: \setlength{\unitlength}{0.62bp}
2356: \put(7,177){\makebox(0,0){\small{$\xh^\EBP$}}}
2357: \put(170,0){\makebox(0,0){\small{$\ih$}}}
2358: \put(-8,-8){\makebox(0,0){\small{$0$}}}
2359: \put(83,-8){\makebox(0,0){\small{$\ih^\MAP$}}}
2360: \put(153,-8){\makebox(0,0){\small{$1$}}}
2361: \put(-8,154){\makebox(0,0){\small{$1$}}}
2362: \put(150,160){\makebox(0,0){\small{A}}}
2363: \put(90,132){\makebox(0,0){\small{B}}}
2364: \put(69,130){\makebox(0,0){\small{C}}}
2365: }
2366: \end{picture}
2367: %\end{center}
2368: \caption{\label{fig:36ebpexit}
2369: (E)BP \exitentropy function $\xh^\EBP(\ih)$.}
2370: \vspace{2bp}
2371: %\end{floatingfigure}
2372: \end{figure}
2373:
2374: Let us apply Theorem \ref{CountingTheorem}.
2375: We start with $\ih_A=1$ (point A).
2376: The residual degree distribution at this point
2377: corresponds of course to the $(3, 6)$-ensemble itself.
2378: As shown in the left-most picture in Fig.~\ref{fig:wd36}, the
2379: corresponding function $\Psi_{\nddp}(u)$ has only a single maximum
2380: at $u=1$ and one can verify that
2381: $\Psi_{\nddp}''(1)<0$. Therefore, by Lemma \ref{WELemma}
2382: we know that with high probability the rate of a randomly chosen element from this
2383: ensemble is close to the design rate.
2384: \begin{figure}[htp]
2385: \centering
2386: \setlength{\unitlength}{1bp}
2387: \begin{picture}(240,70)
2388: \put(10,0){\includegraphics[width=60bp]{WeightDistribution36entire}}
2389: \put(90,0){\includegraphics[width=60bp]{WeightDistribution36052}}
2390: \put(170,0){\includegraphics[width=60bp]{WeightDistribution36MAPth}}
2391: \put(10,0)
2392: {
2393: \put(30,20){\makebox(0,0){A}}
2394: \put(58,-3){\makebox(0,0){$\scriptstyle{3}$}}
2395: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2396: \put(19,-3){\makebox(0,0){$\scriptstyle{1}$}}
2397: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac14}$}}
2398: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac12}$}}
2399: }
2400: \put(90,0)
2401: {
2402: \put(30,20){\makebox(0,0){B}}
2403: \put(58,-3){\makebox(0,0){$\scriptstyle{3}$}}
2404: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2405: \put(19,-3){\makebox(0,0){$\scriptstyle{1}$}}
2406: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac14}$}}
2407: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac12}$}}
2408: }
2409: \put(170,0)
2410: {
2411: \put(30,20){\makebox(0,0){C}}
2412: \put(58,-3){\makebox(0,0){$\scriptstyle{3}$}}
2413: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2414: \put(19,-3){\makebox(0,0){$\scriptstyle{1}$}}
2415: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac14}$}}
2416: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac12}$}}
2417: }
2418: \end{picture}
2419: \caption{
2420: Function $\Psi_{\nddp}(u)$ for the \ddp formed by the residual
2421: ensemble in A, B and C.
2422: }
2423: \label{fig:wd36}
2424: \end{figure}
2425: Next, consider the point $\ih_B=0.52$ (point B). Again, the
2426: conditions are verified, and therefore the conditional entropy
2427: at this point is given by equation (\ref{equ:conditonalentropy}).
2428: We get $H(X \mid Y(\ih_B)) \approx 0.02755$.
2429: Finally, consider the ``critical'' point $\ih_C \approx 0.48815$.
2430: As one can see from the right-most picture in Fig.~\ref{fig:wd36},
2431: this is the point at which a second global maximum appears. Just to the right
2432: of the point the conditions of Theorem \ref{CountingTheorem}
2433: are still fulfilled, whereas to the left of it they are violated.
2434: Further, at this point Eq. (\ref{equ:conditonalentropy})
2435: states that $H(X \mid Y(\ih_C)) = 0$. We conclude that
2436: $\ih^\MAP = \ih_C \approx 0.48815$, confirming our result from
2437: Example \ref{exa:36upperbound}. Since the bound is tight at the MAP
2438: threshold it follows that $\xh^\MAP = \xh^\BP$ for all points ``to the right''
2439: of the MAP threshold (this is true since $\xh^\MAP \leq \xh^\BP$ always,
2440: and the tightness of the bound at the MAP threshold
2441: shows that the area under $\xh^\BP$ is exactly equal to the rate).
2442: We see that in this simple case Theorem \ref{CountingTheorem}
2443: allows us to construct the complete MAP \exitentropy curve.
2444: \eex
2445:
2446:
2447: \bex[Ensemble LDPC($\frac{3x+3x^2+4x^{13}}{10},x^6$)]
2448: \label{equ:doublejumpexample}
2449: %Consider the ensemble LDPC($\frac{3x+3x^2+4x^{13}}{10},x^6$).
2450: %The corresponding
2451: %EPB EXIT curve (shown already in Fig.~\ref{fig:multijump}) is repeated for
2452: %the convenience of the reader in Fig.~\ref{fig:2Jebpexit}.
2453: Consider the ensemble described in Fig.~\ref{fig:multijump}. Its
2454: EPB EXIT curve is repeated for the convenience of the reader in
2455: Fig.~\ref{fig:2Jebpexit}.
2456: The corresponding BP EXIT curve is shown in detail in Fig.~\ref{fig:epsilon}.
2457: A further discussion of this ensemble can be found in Example \ref{ex:running2jumps}.
2458: %\begin{floatingfigure}{115bp}
2459: \begin{figure}
2460: %\vspace{10bp}
2461: \centering
2462: \setlength{\unitlength}{0.5bp}
2463: \begin{picture}(210,230)
2464: \put(19,10)
2465: {
2466: \put(0,0){\includegraphics[scale=0.62]{EXIT2JWD}}
2467: \setlength{\unitlength}{0.62bp}
2468: \put(127,105){\makebox(0,0){\small{$\xh^\EBP$}}}
2469: \put(170,0){\makebox(0,0){\small{$\ih$}}}
2470: \put(-8,-8){\makebox(0,0){\small{$0$}}}
2471: \put(60,-8){\makebox(0,0){\small{$\ih^\MAP$}}}
2472: \put(106,-8){\makebox(0,0){\small{$\ih^{\MAP,2}$}}}
2473: \put(153,-8){\makebox(0,0){\small{$1$}}}
2474: \put(-8,154){\makebox(0,0){\small{$1$}}}
2475: \put(154,160){\makebox(0,0){\small{A$\rightarrow1$}}}
2476: \put(128,130){\makebox(0,0){\small{B}}}
2477: \put(105,142){\makebox(0,0){\small{C}}}
2478: \put(111,77){\makebox(0,0){\small{D}}}
2479: \put(90,83){\makebox(0,0){\small{E}}}
2480: \put(69,70){\makebox(0,0){\small{F}}}
2481: \put(53,54){\makebox(0,0){\small{G}}}
2482: }
2483: \end{picture}
2484: %\end{center}
2485: \caption{\label{fig:2Jebpexit}
2486: (E)BP \exitentropy function $\xh^\EBP(\ih)$.
2487: }
2488: \vspace{2bp}
2489: %\end{floatingfigure}
2490: \end{figure}
2491: \begin{figure}[htp]
2492: \centering
2493: \setlength{\unitlength}{1bp}
2494: \begin{picture}(240,150)
2495: \put(0,80)
2496: {
2497: \put(10,0){\includegraphics[width=60bp]{WeightDistribution2Jentire}}
2498: \put(90,0){\includegraphics[width=60bp]{WeightDistribution2J05313.ps}}
2499: \put(170,0){\includegraphics[width=60bp]{WeightDistribution2J053}}
2500: %\put(170,0)
2501:
2502: \put(10,0)
2503: {
2504: \put(30,20){\makebox(0,0){A}}
2505: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2506: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2507: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2508: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac14}$}}
2509: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac12}$}}
2510: }
2511: \put(90,0)
2512: {
2513: \put(30,20){\makebox(0,0){B}}
2514: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2515: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2516: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2517: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{400}}$}}
2518: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{200}}$}}
2519: }
2520: \put(170,0)
2521: {
2522: \put(30,20){\makebox(0,0){C}}
2523: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2524: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2525: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2526: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{40}}$}}
2527: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{20}}$}}
2528: }
2529: }
2530: \put(0,0)
2531: {
2532:
2533: \put(10,0)
2534: {
2535: \put(0,0){\includegraphics[width=60bp]{WeightDistribution2J05155}}
2536: \put(30,20){\makebox(0,0){E}}
2537: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2538: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2539: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2540: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{40}}$}}
2541: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{20}}$}}
2542: }
2543: \put(90,0)
2544: {
2545: \put(0,0){\includegraphics[width=60bp]{WeightDistribution2J05}}
2546: \put(30,20){\makebox(0,0){F}}
2547: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2548: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2549: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2550: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{40}}$}}
2551: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{20}}$}}
2552: }
2553: \put(170,0)
2554: {
2555: \put(0,0){\includegraphics[width=60bp]{WeightDistribution2JMAPth}}
2556: \put(30,20){\makebox(0,0){G}}
2557: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
2558: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
2559: \put(38,-3){\makebox(0,0){$\scriptstyle{1}$}}
2560: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{40}}$}}
2561: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{20}}$}}
2562: }
2563: }
2564: \end{picture}
2565: \caption{
2566: Function $\Psi_{\nddp}(u)$ for the \ddp formed by the residual
2567: ensemble in A, B, C, E, F and G.
2568: }
2569: \label{fig:2JWD}
2570: \end{figure}
2571: Let us again apply Theorem \ref{CountingTheorem}.
2572: We start with $\ih_A=1$ (point A). The residual
2573: degree distribution corresponds of course to the ensemble itself.
2574: As the top left-most picture in Fig.~\ref{fig:2Jebpexit} shows,
2575: the hypotheses are fulfilled and we conclude again that
2576: with high probability the rate of a randomly chosen element from this
2577: ensemble is close to the design rate which is equal to $\drate\approx0.4872$.
2578: Now decrease $\ih$ smoothly. The conditions of Theorem \ref{CountingTheorem} stay
2579: fulfilled until we get to $\epsilon_B \approx 0.5313$ (point B). At this point
2580: a second global maximum of the function $\Psi_{\nddp}(u)$ occurs.
2581: As the pictures in the bottom row of
2582: Fig.~\ref{fig:2Jebpexit} show, the hypotheses of Theorem \ref{CountingTheorem}
2583: are again fulfilled over the whole segment from E (the
2584: first threshold of the BP decoder corresponding to
2585: %$\ih_E \approx0.5186$
2586: $\ih_E \approx0.5156$) till G. In particular,
2587: at the point G, which corresponds to $\ih_G = \ih^\MAP \approx 0.4913$,
2588: the trial entropy reaches zero, which shows that this
2589: is the MAP threshold.
2590:
2591: We see that for this example Theorem \ref{CountingTheorem} allows
2592: us to construct the MAP EXIT curve for the segment from A to B and the
2593: segment from E to G. Over both these segments we have $\xh^\MAP=\xh^\BP$.
2594: In summary, we can determine the MAP threshold and we see that the
2595: balance condition applies ``at the jump G'' (the MAP threshold).
2596: But the straightforward application of Theorem \ref{CountingTheorem}
2597: does not provide us with a means of determining $\xh^\MAP$
2598: between the points B and D. Intuitively, $\xh^\MAP$ should go from
2599: B to C (which corresponds to $\ih^C \approx 0.5156$).
2600: At this point one would hope that a local balance condition
2601: again applies and that the MAP EXIT curve jumps
2602: to the ``lower branch'' to point D. It should then continue smoothly
2603: until the point G (the MAP threshold) at which it finally jumps to zero.
2604: As we will discuss in more detail in Example \ref{ex:running2jumps}, after our analysis
2605: of the \maxwell decoder, this is indeed true, and $\xh^\MAP$ is
2606: as shown in Fig.~\ref{fig:multijump}.
2607:
2608: Assuming Theorem \ref{CountingTheorem} applies,
2609: we know that at the MAP threshold the matrix corresponding to the residual
2610: graph becomes a full rank square matrix. What happens at the jump
2611: at point C? At this point the matrix corresponding to the residual
2612: graph takes, after some suitable swapping of
2613: columns and rows, the form
2614: \begin{align*}
2615: \left(
2616: \begin{array}{cc}
2617: U & V \\
2618: 0 & W
2619: \end{array}
2620: \right),
2621: \end{align*}
2622: where $W$ is a full rank square matrix of dimension
2623: $\epsilon_C (\lnode(\xr_C)-\lnode(\xr_D))$.
2624: The MAP decoder can therefore solve the part of the equation corresponding to the
2625: submatrix $W$.
2626: \eex
2627: %\bex[Bound Not Tight]
2628: %Code where the threshold is given by stab cond + one jump later.
2629: %%$(\ledge,\redge)=(\frac{2x+3x^6}{5},x^4)$
2630: %%\begin{figure}[hbt]
2631: %%\vspace{5bp}
2632: %%\centering
2633: %%\setlength{\unitlength}{0.5bp}
2634: %%\begin{picture}(360,180)
2635: %%\put(-50,20){\includegraphics[scale=0.5]{stab1Jtemp}}
2636: %%\put(200,20){\includegraphics[scale=0.5]{nottight}}
2637: %%\put(50,0){\makebox(0,0){\footnotesize{(i)} }}
2638: %%\put(250,0){\makebox(0,0){\footnotesize{(ii)} }}
2639: %%\end{picture}
2640: %%\caption{Bound Tightness: Rate and design rate for the residual graph at the fixed-point. (i) The rate is given by the design rate for $\ih\approx0.73$ (ii) It is not the case for $\ih\approx0.701$}
2641: %%\label{fig:}
2642: %%\end{figure}
2643: %\begin{figure}[htp]
2644: %\centering
2645: %\setlength{\unitlength}{1bp}
2646: %\begin{picture}(240,140)
2647: %\put(5,28)
2648: %{
2649: %\put(5,5){\includegraphics[width=100bp]{WeightDistribution2x3x6div5x4-073}}
2650: %\put(135,5){\includegraphics[width=100bp]{WeightDistribution2x3x6div5x4-07}}
2651: % coordinates
2652: %\put(0,0){\makebox(0,0){\small{$0$}}}
2653: %\put(100,0){\makebox(0,0){\small{$1$}}}
2654: %\put(0,100){\makebox(0,0){\small{$1$}}}
2655: %\put(130,0){\makebox(0,0){\small{$0$}}}
2656: %\put(230,0){\makebox(0,0){\small{$1$}}}
2657: %\put(130,100){\makebox(0,0){\small{$1$}}}
2658: %% EXIT in (a)
2659: %\put(40,-5){\makebox(0,0){\small{$\ih^\BP$}}}
2660: % EXIT in (b)
2661: %\put(179,-5){\makebox(0,0){\small{$\ih^\MAP$}}}
2662: %\put(200,80){\makebox(0,0){\small{$~$}}}
2663: %}
2664: % caption
2665: %\put(60,7){\makebox(0,0){{\small{(a)}}}}
2666: %\put(190,7){\makebox(0,0){{\small{(b)}}}}
2667: %\end{picture}
2668: %\caption{
2669: %When
2670: %}
2671: %\label{fig:}
2672: %\end{figure}
2673:
2674: %\eex
2675:
2676: %
2677: %*****************************************************************************
2678: %
2679: \section{Maxwell Construction}
2680: %
2681: \label{MaxwellSection}
2682: The balance condition described in Section \ref{OverviewSection}
2683: and Section \ref{UpperSection} is strongly reminiscent of the
2684: well-known ``Maxwell construction'' in
2685: the theory of phase transitions. This is described briefly
2686: in Fig.~\ref{fig:VanDerWaals}.
2687:
2688: \begin{figure}[htp]
2689: \centering
2690: \setlength{\unitlength}{1bp}
2691: \begin{picture}(240,140)
2692: \put(5,28)
2693: {
2694: \put(5,5){\includegraphics[width=100bp]{VanDerWaals2}}
2695: \put(135,5){\includegraphics[width=100bp]{VanDerWaals1}}
2696: % coordinates
2697: \put(0,0){\makebox(0,0){\small{$0$}}}
2698: \put(100,0){\makebox(0,0){\small{${\scriptstyle{8}}$}}}
2699: \put(9,0){\makebox(0,0){\small{${\scriptscriptstyle{\frac13}}$}}}
2700: \put(0,100){\makebox(0,0){\small{$\scriptstyle2$}}}
2701: \put(130,0){\makebox(0,0){\small{$0$}}}
2702: \put(230,0){\makebox(0,0){\small{$\scriptstyle8$}}}
2703: \put(139,0){\makebox(0,0){\small{${\scriptscriptstyle{\frac13}}$}}}
2704: \put(130,100){\makebox(0,0){\small{$\scriptstyle2$}}}
2705: % VdW in (a)
2706: \put(89,47){\makebox(0,0){\includegraphics[scale=0.16]{EmptyBecher}}}
2707: \put(72,25){$\swarrow$}
2708: \put(39,55){\makebox(0,0){\includegraphics[scale=0.16]{HalfBecher}}}
2709: \put(38,33){$\downarrow$}
2710: \put(22,55){\makebox(0,0){\includegraphics[scale=0.16]{DreiViertelBecher}}}
2711: \put(18,33){$\downarrow$}
2712: \put(20,92){\makebox(0,0){\includegraphics[scale=0.16]{FulledBecher}}}
2713: \put(11,70){$\swarrow$}
2714: \put(4,110){\makebox(0,0){$P$}}
2715: \put(-4,30){\makebox(0,0){$P_c$}}
2716: \put(110,4){\makebox(0,0){$V$}}
2717: % VsW in (b)
2718: \put(179,-5){\makebox(0,0){\small{$~$}}}
2719: \put(200,80){\makebox(0,0){\small{$~$}}}
2720: \put(130,0)
2721: {
2722: \put(4,110){\makebox(0,0){$P$}}
2723: \put(-4,30){\makebox(0,0){$P_c$}}
2724: \put(110,4){\makebox(0,0){$V$}}
2725: }
2726: }
2727: % caption
2728: \put(60,7){\makebox(0,0){{\small{(a)}}}}
2729: \put(190,7){\makebox(0,0){{\small{(b)}}}}
2730: \end{picture}
2731: \caption{
2732: Maxwell construction in thermodynamics.
2733: (a) Pressure-volume diagram for
2734: the liquid-vapor phase transition (b) Van der Waals
2735: curve (using reduced variables, given by
2736: $(p+\frac{2}{V^2})(3V-1)=8T$ at the reduced temperature $T=0.85$ ) and
2737: the Maxwell construction.
2738: Consider the case of a
2739: liquid-gas phase transition of water.
2740: If a small amount of liquid is placed in a completely empty
2741: (and hermetically closed)
2742: large container at room temperature, the water evaporates.
2743: The vapor exerts pressure on the walls of the container.
2744: By gradually reducing the volume of the container,
2745: we increase the vapor pressure $P$ until it reaches a {\em critical} value $P_c$
2746: (which depends on the temperature). At this point the vapor
2747: condensates into liquid water. The pressure stays constant throughout this
2748: transformation.
2749: When there is no space left for the vapor,
2750: the pressure starts to rise again, and as shown in (a) it does so
2751: very quickly (since it is difficult to compress water).
2752: In many theoretical descriptions of this phenomenon, a
2753: non-monotonic pressure-volume curve is obtained like in (b) with
2754: the Van Der Waals model. The Maxwell construction
2755: allows to modify the ``unphysical'' part of this curve and to obtain a
2756: consistent result. We want to join the two decreasing branches of
2757: the theoretical curve with a constant-pressure line, as observed in experiments.
2758: At which height should we placed the horizontal
2759: line? The basic idea of the Maxwell construction is that, at the critical
2760: pressure $P_c$, the vapor and the liquid are in {``{equilibrium}''}.
2761: This means that we can transform an infinitesimal quantity of vapor
2762: into liquid (or vice versa) without doing any {``{work}''} on the system.
2763: Because of this reason, the vapor begins its transformation into
2764: liquid at $P_c$. The work done on the system in an infinitesimal
2765: transformation is $P\, \text{d}V$, where $\text{d}V$ represents the
2766: variation of the volume. Using this fact, it can be shown that the
2767: above equilibrium condition implies the equality of the areas
2768: of the two regions between the horizontal line and the original
2769: non-monotonous pressure-volume curve. See, e.g.,~\cite{KiK02}.
2770: }
2771: \label{fig:VanDerWaals}
2772: \end{figure}
2773:
2774:
2775: %
2776: %**********************************************************************
2777: %
2778: \subsection{Maxwell Decoder}
2779: \label{MaxwellIntro}
2780:
2781: Inspired by the statistical mechanics analogy, we will explain the
2782: balance condition (shown on the right in Fig.~\ref{fig:exitcurve}) which determines
2783: the MAP threshold by analyzing a ``BP decoder with guessing''.
2784: The state of the algorithm can be associated to a point moving
2785: along the EBP \exitentropy curve. The evolution starts at the
2786: point of full entropy and ends at zero entropy.
2787: The analysis of this algorithm is also most conveniently
2788: phrased in terms of the EBP \exitentropy curve and implies a proof of
2789: Theorem~\ref{ImprovedUB}.
2790: Because of this balance condition we term this decoding algorithm
2791: the Maxwell (M) decoder.
2792: Note that a similar algorithm is discussed in \cite{PiF04}
2793: although it is motivated by some more practical concerns.
2794:
2795: Analogously to the usual BP decoder for the erasure channel,
2796: the \maxwell
2797: decoder admits two equivalent descriptions: either as a {\em sequential}
2798: (i.e., bit-by-bit in the spirit of \cite{LMSS01b}) or as a {\em message-passing}
2799: algorithm. While the former approach is more intuitive,
2800: the latter allows for a simpler analysis.
2801: We shall first describe the \maxwell decoder as a sequential
2802: procedure and sketch the main features of its behavior. In the next section
2803: we will turn to a message-passing setting and complete its analysis.
2804:
2805: \begin{figure*}[hbt]
2806: \begin{center}
2807: \hfil
2808: \newlength{\FigLength}
2809: \setlength{\FigLength}{150bp}
2810: \setlength{\unitlength}{0.625bp}
2811: \begin{tabular}{lll}
2812: \input{./maxwelldecodertutorial/residu1} &
2813: \input{./maxwelldecodertutorial/residu2} &
2814: \input{./maxwelldecodertutorial/residu3} \\
2815: {\footnotesize (i) Unknown bits after transmission}
2816: & {\footnotesize (ii) Decoding bit 1 from check 1}
2817: & {\footnotesize(iii) Decoding bit 10 from check 5} \\
2818: \input{./maxwelldecodertutorial/residu4} &
2819: \input{./maxwelldecodertutorial/residu5} &
2820: \input{./maxwelldecodertutorial/residu6} \\
2821: {\footnotesize(iv) Decoding bit 11 from check 8}
2822: & {\footnotesize(v) Guessing bit 2}
2823: & {\footnotesize(vi) Guessing bit 6} \\
2824: \input{./maxwelldecodertutorial/residu7} &
2825: \input{./maxwelldecodertutorial/residu8} &
2826: \input{./maxwelldecodertutorial/residu9} \\
2827: {\footnotesize(vii) Decoding bit 28 ({\footnotesize{$x_2+x_6$}}) from check 6}
2828: & {\footnotesize(viii) Decoding bit 19 ({\footnotesize{$x_6$}}) from check 14}
2829: & {\footnotesize(ix) Guessing bit 12} \\
2830: \input{./maxwelldecodertutorial/residu10} &
2831: \input{./maxwelldecodertutorial/residu11} &
2832: \input{./maxwelldecodertutorial/residu12} \\
2833: {\footnotesize(x) Decoding bit 30 ({\footnotesize{$x_6+x_{12}=x_{12}$}})}
2834: & {\footnotesize(xi) Decoding bit 24 ({\footnotesize{$x_{12}$}}) from check 3}
2835: & {\footnotesize(xii) Decoding bit 23 ({\footnotesize{$x_2+x_{12}$}}) from } \\
2836: {\footnotesize ~}
2837: & {\footnotesize from checks 11 and 15 $\longrightarrow x_6=0$}
2838: & {\footnotesize check 4} \\
2839: \input{./maxwelldecodertutorial/residu13} &
2840: \input{./maxwelldecodertutorial/residu14} & \\
2841: {\footnotesize(xiii) Decoding bit 21 ({\footnotesize{$x_2+x_{12}$}}) from check 7}
2842: & {\footnotesize(xiv) Decoding bit 29 ({\footnotesize{$x_{12}=x_2+x_{12}$}}) }
2843: & {\footnotesize(xv) Decoding bit 26 $\longrightarrow x_{12}=0$ } \\
2844: {\footnotesize ~}
2845: & {\footnotesize from checks 2 and 9 $\longrightarrow x_2=0$}
2846: & {\footnotesize Final step! } \\
2847: \end{tabular}
2848: \caption{\label{fig:maxwelldecoder} \maxwell decoder applied to a simple
2849: example: a $(3,6)$ LDPC code of length $\n=30$. Assume that the all-zero
2850: codeword has been transmitted. At the decoder, the
2851: received (i.e., known and equal to $0$) bits are removed from the
2852: bipartite graph. The remaining graph is shown in (i). The first
2853: phase is the standard BP algorithm: in the first three steps, the decoder
2854: proceeds as the standard BP decoder and determines the bits $1$, $10$ and
2855: $11$, until it gets stuck in a stopping set shown in (iv). The second
2856: phase is distinct to the \maxwell decoder: it is the guessing/contradiction
2857: phase. The decoder guesses the (randomly chosen) bit $2$: this means that
2858: it creates two simultaneously running copies, one which proceeds under
2859: the assumption that bit $2$ takes the value $0$, the other which assumes
2860: that this bit takes the value $1$. The decoder
2861: then proceeds as the standard BP algorithm. Any time it gets stuck, it
2862: guesses a new bit and duplicates the number of simultaneously running copies.
2863: This process continues until a contradiction occurs, e.g., at the
2864: 9$^{\text{th}}$ step $(ix)$: the variable node $x_{30}$ (either
2865: $x_{30}=0$ or $x_{30}=1$ depending of which copy we are considering)
2866: is connected to two check nodes
2867: of degree one. The incoming messages from those nodes are $x_6+x_{12}$
2868: and $x_{12}$, respectively.
2869: Consistency now requires that $x_6+x_{12}=x_{12}$, i.e., that $x_6=0$,
2870: such that only the decoding copies corresponding to $x_6=0$ survive.
2871: Phases of guessing and phases of standard BP decoding might alternate.
2872: Decoding is successful (in the sense that a MAP decoder would
2873: have succeeded) if only a single copy survives at the very end
2874: of the decoding process. ``Contradictions'' can be seen as ``confirmations'' %or
2875: %``resolutions of equations''
2876: or ``conditions'' in this message-passing setting.
2877: }
2878: \end{center}
2879: \hrulefill
2880: \vspace*{4pt}
2881: \end{figure*}
2882: %
2883: %
2884: %
2885: Given the received word which was
2886: transmitted over BEC$(\ih)$, the decoder proceeds iteratively as does
2887: the standard BP decoder. At each time step a parity-check equation
2888: involving a single undetermined variable is chosen and used to
2889: determine the value of the variable. This value is substituted in any
2890: parity-check equation involving the same variable.
2891: If at any time the iterative
2892: decoding process gets stuck in a non-empty stopping set,
2893: a position $i\in [\n]$ is chosen uniformly at random. The decoder is
2894: said to {\em guess} a bit.
2895: If the bit associated to this position is not known yet,
2896: the decoder replicates\footnote{Here we describe the decoder
2897: as a `breadth-first' search procedure: at each bifurcation we explore
2898: in parallel all the available options.
2899: One can easily construct an equivalent `depth-first' search:
2900: first take a complete sequence of choices and, if no codeword is
2901: found, backtrack.} any running copy of the decoding process,
2902: and it proceeds by running one copy of each process
2903: under the assumption
2904: that $x_i=\0$ and the other one under the assumption that $x_i=\1$.
2905:
2906: It can happen that during the decoding process a variable
2907: receives non-erased messages from several check nodes.
2908: In such a case, these messages can be distinct and, therefore,
2909: inconsistent.
2910: Such an event is termed a {\em contradiction}.
2911: Any running copy of the decoding process which encounters a contradiction
2912: terminates.
2913: The decoding process finishes once all bits have been determined.
2914: At this point, each surviving copy outputs the determined word.
2915: Each such word is by construction a codeword which is
2916: compatible with the received information.
2917: Vice versa, for each codeword which is compatible with
2918: the received information, there will be a surviving copy.
2919: In other words, the \maxwell decoder performs a complete {\em list decoding} of
2920: the received message. Fig.~\ref{fig:maxwelldecoder} shows
2921: the workings of the \maxwell decoder by means of a specific example.
2922: The corresponding instance of the decoding process is depicted in
2923: Fig.~\ref{fig:maxwelltwodecoder} from the
2924: perspective of the various simultaneous copies.
2925:
2926: Let us briefly describe how the analysis of the above algorithm
2927: is related to the balance condition and the proof of
2928: Theorem~\ref{ImprovedUB}.
2929: Instead of explaining the balance between the areas as shown in
2930: Fig.~\ref{fig:exitcurve}, we consider the balance of the two areas shown
2931: in Fig.~\ref{fig:exitbalance}. Note that these two areas differ from the
2932: previous ones only by a common term, so that the condition for balance stays
2933: unchanged.
2934: From the above description it follows that at any given time $\iter$
2935: there are $2^{\entropy(\iter)}$ copies running,
2936: where $\entropy(\iter)$ is a natural number which evolves with time.
2937: In fact, each time a bit is guessed, the number of copies is doubled,
2938: while it is halved each time a contradiction occurs.
2939: Call $\iter_{\text{out}}$ the time at which all transmitted bits have
2940: been determined and the list of decoded words is output
2941: ($\iter_{\text{out}}$ does not depend upon the particular copy of the process in
2942: consideration). Since the \maxwell decoder is a complete list decoder
2943: and since all output codewords have equal posterior probability,
2944: $H(X|Y) = \entropy(\iter_{\text{out}})$. On the other hand, $\entropy(\iter_{\text{out}})$
2945: is equal to the total number of guesses minus the total number of
2946: contradictions which occurred during the evolution of the algorithm.
2947: As we will see in greater detail in the next section,
2948: the total number of guesses divided by $n$ converges to the area of the
2949: dark gray region
2950: in Fig.~\ref{fig:exitbalance} (a), while the total number of contradictions
2951: divided by $n$ is asymptotically not larger than the dark gray area
2952: in Fig.~\ref{fig:exitbalance} (b). Therefore, as long as
2953: $\cp$ is strictly larger than the value at which we have balance,
2954: call this value $\overline{\cp}^\MAP$,
2955: $\lim_{\n\to\infty}\frac{\expectation[H(X|Y(\ih))]}{\n}>0$. This implies that $\overline{\cp}^\MAP\ge\cp^\MAP$.
2956:
2957: We expect that the number of
2958: contradictions divided by $\n$ is indeed asymptotically equal to
2959: the dark gray area in Fig.~\ref{fig:exitbalance} (b).
2960: Although we are not able to prove this statement in full generality,
2961: it follows from Theorem~\ref{CountingTheorem}, whenever the hypotheses hold.
2962:
2963: \begin{figure}[htp]
2964: %\begin{center}
2965: \centering
2966: \setlength{\unitlength}{0.8bp}
2967: \newlength{\longueur}
2968: \setlength{\longueur}{240bp}
2969: \newcounter{Subdi}
2970: \setcounter{Subdi}{20}
2971: \newcounter{Step}
2972: \setcounter{Step}{19} % change step if needed
2973: \newcounter{Shift}
2974: \setcounter{Shift}{0}
2975: %
2976: \begin{picture}(300,120)
2977: \put(4,0)
2978: {
2979: \put(0,0){\includegraphics[width=\longueur]{explosion}}
2980: \put(-3,38){\begin{turn}{0}\makebox(0,0){{\small $\entropy(\iter)$} }\end{turn}}
2981: \put(300,5){\begin{turn}{0}\makebox(0,0){{\small $\iter$} }\end{turn}}
2982: % labelizing x-axis
2983: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_1$} }\end{turn}}
2984: \addtocounter{Subdi}{\value{Step}}
2985: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{10}$} }\end{turn}}
2986: \addtocounter{Subdi}{\value{Step}}
2987: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{11}$} }\end{turn}}
2988: \addtocounter{Subdi}{\value{Step}}
2989: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{2}$} }\end{turn}}
2990: \addtocounter{Subdi}{\value{Step}}
2991: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{6}$} }\end{turn}}
2992: \addtocounter{Subdi}{\value{Step}}
2993: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{28}$} }\end{turn}}
2994: \addtocounter{Subdi}{\value{Step}}
2995: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{19}$} }\end{turn}}
2996: \addtocounter{Subdi}{\value{Step}}
2997: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{12}$} }\end{turn}}
2998: \addtocounter{Subdi}{\value{Step}}
2999: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{30}$} }\end{turn}}
3000: \addtocounter{Subdi}{\value{Step}}
3001: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{24}$} }\end{turn}}
3002: \addtocounter{Subdi}{\value{Step}}
3003: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{23}$} }\end{turn}}
3004: \addtocounter{Subdi}{\value{Step}}
3005: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{21}$} }\end{turn}}
3006: \addtocounter{Subdi}{\value{Step}}
3007: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{29}$} }\end{turn}}
3008: \addtocounter{Subdi}{\value{Step}}
3009: \put(\value{Subdi},0){\begin{turn}{0}\makebox(0,0){{\small $x_{26}$} }\end{turn}}
3010: \addtocounter{Subdi}{\value{Step}}
3011: %
3012: \setcounter{Subdi}{20}
3013: \addtocounter{Shift}{85}
3014: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3015: \addtocounter{Subdi}{\value{Step}}
3016: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3017: \addtocounter{Subdi}{\value{Step}}
3018: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3019: \addtocounter{Subdi}{\value{Step}}
3020: \addtocounter{Shift}{15}
3021: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3022: \addtocounter{Shift}{-44}
3023: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3024: \addtocounter{Subdi}{\value{Step}}
3025: \addtocounter{Shift}{62}
3026: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3027: \addtocounter{Shift}{-28}
3028: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3029: \addtocounter{Shift}{-24}
3030: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3031: \addtocounter{Shift}{-27}
3032: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3033: \addtocounter{Subdi}{\value{Step}}
3034: \addtocounter{Shift}{86}
3035: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3036: \addtocounter{Shift}{-40}
3037: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3038: \addtocounter{Shift}{-13}
3039: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3040: \addtocounter{Shift}{-40}
3041: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3042: \addtocounter{Subdi}{\value{Step}}
3043: \addtocounter{Shift}{93}
3044: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3045: \addtocounter{Shift}{-40}
3046: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3047: \addtocounter{Shift}{-13}
3048: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3049: \addtocounter{Shift}{-40}
3050: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3051: \addtocounter{Subdi}{\value{Step}}
3052: \addtocounter{Shift}{93}
3053: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3054: \addtocounter{Shift}{-15}
3055: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3056: \addtocounter{Shift}{-12}
3057: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3058: \addtocounter{Shift}{-13}
3059: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3060: \addtocounter{Shift}{-13}
3061: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3062: \addtocounter{Shift}{-12}
3063: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3064: \addtocounter{Shift}{-15}
3065: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3066: \addtocounter{Shift}{-13}
3067: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3068: \addtocounter{Subdi}{\value{Step}}
3069: \addtocounter{Shift}{93}
3070: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3071: \addtocounter{Shift}{-15}
3072: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3073: \addtocounter{Shift}{-12}
3074: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3075: \addtocounter{Shift}{-13}
3076: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3077: \addtocounter{Shift}{-13}
3078: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3079: \addtocounter{Shift}{-12}
3080: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3081: \addtocounter{Shift}{-15}
3082: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3083: \addtocounter{Shift}{-13}
3084: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3085: \addtocounter{Shift}{93}
3086: \addtocounter{Subdi}{\value{Step}}
3087: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3088: \addtocounter{Shift}{-15}
3089: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3090: \addtocounter{Shift}{-12}
3091: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3092: \addtocounter{Shift}{-13}
3093: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3094: \addtocounter{Shift}{-13}
3095: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3096: \addtocounter{Shift}{-12}
3097: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3098: \addtocounter{Shift}{-15}
3099: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3100: \addtocounter{Shift}{-13}
3101: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3102: \addtocounter{Subdi}{\value{Step}}
3103: \addtocounter{Shift}{93}
3104: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3105: \addtocounter{Shift}{-15}
3106: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3107: \addtocounter{Shift}{-12}
3108: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3109: \addtocounter{Shift}{-13}
3110: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3111: \addtocounter{Shift}{-13}
3112: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3113: \addtocounter{Shift}{-12}
3114: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3115: \addtocounter{Shift}{-15}
3116: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3117: \addtocounter{Shift}{-13}
3118: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3119: \addtocounter{Subdi}{\value{Step}}
3120: \addtocounter{Shift}{93}
3121: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3122: \addtocounter{Shift}{-15}
3123: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3124: \addtocounter{Shift}{-12}
3125: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3126: \addtocounter{Shift}{-13}
3127: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3128: \addtocounter{Shift}{-13}
3129: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3130: \addtocounter{Shift}{-12}
3131: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3132: \addtocounter{Shift}{-15}
3133: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3134: \addtocounter{Shift}{-13}
3135: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3136: \addtocounter{Shift}{93}
3137: \addtocounter{Subdi}{\value{Step}}
3138: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3139: \addtocounter{Shift}{-15}
3140: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3141: \addtocounter{Shift}{-12}
3142: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3143: \addtocounter{Shift}{-13}
3144: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3145: \addtocounter{Shift}{-13}
3146: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3147: \addtocounter{Shift}{-12}
3148: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3149: \addtocounter{Shift}{-15}
3150: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $1$} }\end{turn}}
3151: \addtocounter{Shift}{-13}
3152: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3153: \addtocounter{Shift}{93}
3154: \addtocounter{Subdi}{\value{Step}}
3155: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3156: \addtocounter{Shift}{-15}
3157: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3158: \addtocounter{Shift}{-12}
3159: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3160: \addtocounter{Shift}{-13}
3161: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3162: \addtocounter{Shift}{-13}
3163: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3164: \addtocounter{Shift}{-12}
3165: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3166: \addtocounter{Shift}{-15}
3167: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $~$} }\end{turn}}
3168: \addtocounter{Shift}{-13}
3169: \put(\value{Subdi},\value{Shift}){\begin{turn}{0}\makebox(0,0){{\small $0$} }\end{turn}}
3170: \addtocounter{Subdi}{\value{Step}}
3171: }
3172: \end{picture}
3173: \caption{ \maxwell decoder applied to the simple
3174: example shown in Fig. \ref{fig:maxwelldecoder}. The all-zero codeword is decoded.
3175: The initial phase
3176: coincides with standard message-passing BP algorithm:
3177: a single copy of the process decodes a bit at a time.
3178: After three steps, the BP decoder gets stuck in a stopping
3179: set and several steps of guessing follow. During this phase
3180: the associated entropy $\entropy(\iter)$ increases.
3181: After this guessing phase, the standard message
3182: passing phase resumes. More and more copies terminate due to
3183: inconsistent messages arriving at variable nodes.
3184: At the end only one copy survives. This shows that this example
3185: has a unique MAP solution.
3186: }
3187: \label{fig:maxwelltwodecoder}
3188: %\end{center}
3189: \end{figure}
3190:
3191: %
3192: %\subsection{New Analysis - December 7, 2004}
3193: %Consider a graph $\graph$ of length $\n$ and transmission over BEC($\ih$).
3194: %Assuming decoding is performed using the iterative \maxwell decoder.
3195: %Initially, in average, there are $\n(1-\ih)$ known bits uniformly
3196: %distributed in a set of known bits.
3197: %This number, which we denote $\n \tau$, increases during the decoding
3198: %process until it reaches the $\n$ bits. At any point in time, those $\n \tau$ bits
3199: %are uniformly distributed in a set which we denote $K_{\tau}$, $|K_{\tau}|=\n \tau$.
3200: %Let $\Gamma=\n \gamma$ denote the number of guesses at the same point in time.
3201: %At any point in time $\tau$,
3202: %(ASSUMING NO CONTRADICTION APPEARS)
3203: % we further get
3204: %\begin{align*}
3205: %\Gamma(\n \tau+1)=
3206: %\begin{cases}
3207: %\Gamma(\n \tau)+1 , & \text{with~probability~$\frac{|S_\tau|}{\n}$} \\
3208: %\Gamma(\n \tau) , & \text{with~probability~$1-\frac{|S_\tau|}{\n}$},
3209: %\end{cases}
3210: %\end{align*}
3211: %where $S_\tau$ denote the largest stopping set contained in the
3212: %residual graph at this point in time.
3213: %(ASSUMING $\Gamma$ CONCENTRATES).
3214: %In the asymptotic limit, when the blocklength $\n \rightarrow \infty$, we get
3215: %$$
3216: %\gamma'(\tau)= \lim_{\n \rightarrow \infty} \frac{S_\tau}{\n}=\ih(\tau) \xh^\BP(\ih(\tau)).
3217: %$$
3218: %Since $\ih'(\tau)=-\ih(\tau)$, we then have
3219: %$$
3220: %\expectation\left[\gamma \right] = \int_{guessing phase with no contradiction} \xh(\ih)\text{d} \ih
3221: %$$
3222: %Now,
3223: %\begin{align*}
3224: %\Pr\left\{\text{contradiction} \right\} &= \Pr\{ \ell^{(1)}_{max}\geq \ell_1 \}\\
3225: %& \leq \Pr\{\ell^{(1)}_{max}\geq \ell^*, \ell_1 \geq \ell^* \}+ \Pr\{\ell_1 < \ell^* \}
3226: %\end{align*}
3227: %
3228: %PB: will not include the case of multiple jumps.
3229: %Additional difficulty to prove all concentration theorems.
3230: % Tree analysis would be nicer
3231: %
3232: %***************************************************************
3233: %
3234: \subsection{Message-Passing Setting}
3235: \label{MessagePassingSection}
3236: We describe now a message-passing algorithm that is equivalent
3237: to the above sequential formulation. First note that because
3238: of the code linearity, the symmetries of the channel and the decoding algorithm,
3239: we can simplify our analysis by making the all-zero codeword assumption,
3240: see \cite{RiU01}.
3241:
3242: We assign a label $\chm{i}$ to the variable node of
3243: index $i$. The label can take three possible values
3244: $\chm{i}\in\{\0, \?, \g\}$.
3245: It can be viewed as the output of some fictitious channel,
3246: and indicates how the algorithm is going to treat
3247: that variable node. The fictitious channel is memoryless: each
3248: variable node is assigned a
3249: $\0$ with probability $1-\ih$, a $\?$ with probability $\ih(1-\gp)$
3250: and a $\g$ with probability $\ih \gp$.
3251: %The meaning of $\0$ and $\?$
3252: %is the same as in the original BP decoding algorithm.
3253: The parameter
3254: $\gp$ represents the fraction of
3255: % potential
3256: {\em guesses} ventured so far.
3257: %\footnote{It may be that a bit $i$ with $\m^\cp_i=\g$ is rather determined
3258: %in terms of the others and therefore not guessed.}
3259: %Whenever the algorithm
3260: %encounters a bit labeled $\g$, the \maxwell decoder tries to resolve
3261: %(in the sense of linear systems) with respect to the corresponding variable.
3262:
3263: The new message-passing algorithm employs left-to-right messages $\lrm{}$
3264: and right-to-left messages $\rlm{}$, all of which take values in $\{\0,\?,\g\}$.
3265: The meaning of the $0$ message and the $\?$ message is the same as for the
3266: BP algorithm. A $\g$ message indicates that either the bit from which this
3267: message emanates has been guessed or that the value of this bit
3268: can be expressed as a linear combination of other bit values which have
3269: been guessed.
3270: %The meaning of a message referring to the $i^{\text{th}}$ variable node and
3271: %carrying value $\g$ is that no sure information is known about the
3272: %$i^{\text{th}}$ bit, but rather its expression in terms of other variables
3273: %of indices $j$ for which $\chm{j}=\g$.
3274: Operationally, we can think of the message $\mu_i=\g$
3275: as being a shorthand
3276: for a non-empty list of indices $\List_i = \{j_1,\dots, j_k\}$.
3277: This list indicates that $x_i$ is expressible as $x_i= x_{j_1}+\cdots +x_{j_k}$,
3278: where $\{x_{j_1}, \cdots, x_{j_k}\}$ is a set of guessed bits.
3279: %meaning that, as a consequence of the channel output
3280: %(to the variable nodes associated to
3281: %channel outputs $\0$), it is known that the value of the $i^{\text{th}}$ can
3282: %be expressed as $x_i= x_{j_1}+\cdots +x_{j_k}$.
3283:
3284: This motivates the following update
3285: rules for the parity-check and variable nodes shown in Fig.~\ref{fig:updatecnodemaxwell}.
3286: \begin{figure}[hbt]
3287: \vspace{5bp}
3288: \centering
3289: \setlength{\unitlength}{0.5bp}
3290: \begin{picture}(360,180)
3291: \put(0,20){\includegraphics[scale=0.5]{updatecnodemaxwell}}
3292: \put(200,20){\includegraphics[scale=0.5]{updatevnodemaxwell}}
3293: \put(155,100){\makebox(0,0){$\rlm{}$}}
3294: \put(355,100){\makebox(0,0){$\lrm{}$}}
3295: \put(5,140){\makebox(0,0){$\mu_{1}$}}
3296: \put(5,120){\makebox(0,0){$\mu_{2}$}}
3297: \put(5,60){\makebox(0,0){$\mu_{\rdegree-1}$}}
3298: \put(205,140){\makebox(0,0){$\mu_{1}$}}
3299: \put(205,60){\makebox(0,0){$\mu_{\ldegree-1}$}}
3300: \put(280,50){\makebox(0,0){$\chm{}$}}
3301: \put(50,0){\makebox(0,0){\footnotesize{(i)} }}
3302: \put(250,0){\makebox(0,0){\footnotesize{(ii)} }}
3303: \end{picture}
3304: \caption{Update rule for parity-check nodes (i) and variable nodes (ii).}
3305: \label{fig:updatecnodemaxwell}
3306: \end{figure}
3307:
3308: (i) Update rule for a parity-check node of degree $\rdegree$:
3309: Assume that the index set for the $(\rdegree-1)$ messages which enter
3310: the check node is ${\cal{R}}= [\rdegree-1]$.
3311: %(we can think of ${\cal R}$ as either a subset of
3312: %the variable node indices or a subset of the edge indices).
3313: Then
3314: \begin{align*}
3315: \rlm{} =
3316: \begin{cases}
3317: \0, & \text{if~$\forall i\in {\cal{R}}, ~\m_{i}=\0$}, \\
3318: \?, & \text{if~$\exists i\in {\cal{R}}, ~\m_{i}=\?$}, \\
3319: \g, & \text{if~$\forall j\in {\cal{R}}, ~\m_{j} \neq \?$, and
3320: $\exists i\in {\cal{R}}, ~\m_{i}=\g$}.\\
3321: \end{cases}
3322: \end{align*}
3323: With respect to the BP decoder, the only new rule is the one which
3324: leads to $\rlm{} = \g$. It is motivated as follows.
3325: Assume that for all $i \in {\cal{R}}$ we have
3326: $\lrm{i}=0\backslash \g$ and that at least one such message is $\g$.
3327: This means that the connected variables $x_i$, $i \in {\cal{R}}$, are either
3328: known, have been guessed themselves, or can be expressed as
3329: a linear combination of guessed bits (and at least one such value
3330: is indeed either a guess itself or expressible as a linear combination
3331: of guesses).
3332: Since the variable connected to the outgoing edge is the sum of the
3333: variables connected to the incoming edges, it follows that this variable
3334: is also expressible as a linear combination of guesses. Therefore,
3335: $\rlm{}=\g$ in this case.
3336: Operationally, we have $\rdegree-1$
3337: lists $\List_1,\dots,\List_{\rdegree-1}$ (at least one of
3338: which is non-empty) entering the check node.
3339: The outgoing list $\List^{\tt y}$ is obtained as the {\em union} of the incoming lists,
3340: where indices which occur an even number of times in the incoming lists are eliminated.
3341: The list $\List^{\tt y}$ provides a resolution rule for
3342: $x_{1}+\cdots +x_{\rdegree-1}$, and therefore for the variable connected to the
3343: outgoing edge.
3344:
3345: In the above description and the definition of the message-passing rules
3346: we have ignored the possibility that the union
3347: of the incoming lists (at least one of which is non-empty) is empty.
3348: This can happen if a complete cancellation occurs (every index appears
3349: an even number of times in the incoming lists).
3350: %In defining the above rule, we assume that $\List^{\tt y}$ is non-empty
3351: %and therefore correspond to $\m^{\tt y} = \?$. However, it may happen that
3352: %a compete cancellation occurs and the list $\List^{\tt y}$ is empty despite
3353: %$\m^{\tt y} = \?$.
3354: Fortunately, as we shall see, this assumption
3355: has no influence on the proof of Theorem~\ref{ImprovedUB}.
3356:
3357: (ii) Update rule for a variable node of degree $\ldegree$:
3358: Assume that the index set for the $\ldegree-1$ messages
3359: which enter the variable node is ${\cal{L}}= [\ldegree-1]\cup \{ {\tt \ih}\}$.
3360: %(we can think of ${\cal L}$ as either a subset of the check node
3361: %indices or a subset of the edge indices with the extra index ${\tt \ih}$ representing
3362: %the received value).
3363: Then
3364: \begin{align*}
3365: \lrm{} =
3366: \begin{cases}
3367: \0, & \text{if~$\exists i\in {\cal{L}}, ~\m_{i}=\0$}, \\
3368: \?, & \text{if~$\forall i\in {\cal{L}}, ~\m_{i}=\?$}, \\
3369: \g, & \text{if~$\forall i\in {\cal{L}}, ~\m_{i} \neq \0$ and $\exists j\in {\cal{L}}, ~\m_{j}=\g$}.
3370: \end{cases}
3371: \end{align*}
3372: Once again, it should be enough to motivate the rule which leads to
3373: $\lrm{} = \g$. Recall that $\g$ indicates that the bit is not known but
3374: that it has either been guessed or that the bit is expressible as a linear combination
3375: of guessed bits. Therefore, if none of the incoming messages is a $0$,
3376: and at least one is a $\g$, then the outgoing message is a $\g$.
3377: Operationally, this means that the outgoing list is equal to {\em one}
3378: of the incoming non-empty lists. E.g., if the bit itself has been guessed
3379: (i.e., $\chm{i}=\g$) and all other incoming
3380: messages are $\?$ then the outgoing message is $\{i\}$.
3381:
3382: %The message outgoing from the $i^{\text{th}}$ variable node can be resolution
3383: %only if no sure information is known about the $i^{\text{th}}$ variable
3384: %from incoming messages or the received signal.
3385: %Two cases may occur. Either some of the incoming messages indicate
3386: %that $x_i$ can be resolved with respect to other variables.
3387: %In this case the outgoing message will be any
3388: %of these resolutions and the
3389: %variable $x_i$ has been effectively eliminated from the linear system.
3390: %Operationally, one of the incoming lists is copied to the node output.
3391: %In the opposite case, and if $\chm{i}=\g$, then the algorithm will
3392: %resolve with respect to $i$. Operationally, the list $\{i\}$ is
3393: %the outgoing message.
3394:
3395: From the messages we can obtain estimates $\nu_i$, $\in [n]$,
3396: of the transmitted bits (the $\nu_i$'s are node- rather than
3397: edge-quantities).
3398: In order to obtain these estimates we apply
3399: the same rule as for the variable node update, see (ii)
3400: above, with incoming messages corresponding to {\em all} of the
3401: neighboring check nodes. In other words, for a degree $\ldegree$ variable
3402: node, we have ${\cal L} = [\ldegree]\cup\{\epsilon\}$
3403: instead of ${\cal L} = [\ldegree-1]\cup\{\epsilon\}$.
3404:
3405: The consistency of the estimates
3406: implies a set of linear {\em conditions} on the guessed variables.
3407: Consider all the messages $\mu_i$ entering a fixed variable node
3408: and the associated (possibly empty) lists $\List_{i}=\{j^i_1, \cdots, j^i_k\}$.
3409: Let ${\cal L}_{\m}$, $\m\in\{\0,\g,\?\}$ denote the subsets of indices
3410: $i$ with $\m_i=\m$.
3411: %
3412: \begin{enumerate}
3413: %
3414: \item If ${\cal L}_{\0}\neq\emptyset$ and ${\cal L}_{\g}\neq\emptyset$,
3415: then, for any $i\in {\cal L}_{\g}$, we have the condition
3416: %
3417: \begin{eqnarray}
3418: %
3419: x_{j^i_1}+\cdots +x_{j^i_k} = 0\, ,\;\;\;\;\;\;\;\;{\rm mod}\; 2\, .
3420: \label{ConditionA}
3421: %
3422: \end{eqnarray}
3423: %
3424: %where $\List_i \equiv\{j^i_1,\dots, j^i_k\}$.
3425: The total number
3426: of resulting conditions is $|{\cal L}_{\g}|$.
3427: %
3428: \item If ${\cal L}_{\0} = \emptyset$ and $|{\cal L}_{\g}|\ge 2$,
3429: then fix $i\in {\cal L}_{\g}$. For any $l\in {\cal L}_{\g}\backslash \{i\}$,
3430: we have the condition
3431: %
3432: \begin{eqnarray}
3433: %
3434: x_{j^i_1}+\cdots +x_{j^i_k} = x_{j^l_1}+\cdots +x_{j^l_k}\, ,\;\;\;\;\;\;{\rm mod}\; 2\, .
3435: \label{ConditionB}
3436: %
3437: \end{eqnarray}
3438: %
3439: %where $\List_{i} \equiv\{j^i_1,\dots, j^i_k\}$,
3440: %$\List_{l} \equiv\{j^l_1,\dots, j^l_k\}$.
3441: The total number of resulting conditions is $|{\cal L}_{\g}|-1$.
3442: %
3443: \end{enumerate}
3444: %
3445: The algorithm stores in memory each new condition produced during its
3446: execution. Notice that each conditions involves uniquely bits $x_i$
3447: for which $\mu_i^{\epsilon}=\g$. It can happen that
3448: a particular condition is either linearly dependent upon previous
3449: ones or empty. The last case occurs if the corresponding lists are empty,
3450: which in turn may be the consequence of a previous parity-check node update
3451: (see the description of the check-node update rule above).
3452: Given a set of guesses, any subset of them whose values can be chosen
3453: freely without violating any of the conditions produced by
3454: the \maxwell decoder, is said to be {\em independent}.
3455: Of course, the maximal number of independent guesses is equal to the number of
3456: guesses minus the number of linearly independent conditions.
3457:
3458: {\em Conditions} are equivalent, in the present setting
3459: to what have been called contradictions in the description of
3460: of Sec.~\ref{MaxwellIntro}. In fact, if one thinks of guessed bits
3461: as i.i.d. uniformly random in $\{\0,\1\}$ then each new,
3462: independent condition, cf.
3463: Eqs.~(\ref{ConditionA}), (\ref{ConditionB}) is satisfied with
3464: probability $1/2$.
3465:
3466: It is useful to estabilish the following convention for denoting the
3467: successive message passing iterations. At the $t^{\text{th}}$ iteration
3468: (with $t=0,1,\dots$) we first update all the left-to-right messages and
3469: then all the right-to-left messages. We have therefore
3470: $\dots\to\rlm{}(t-1)\to\lrm{}(t)\to\rlm{}(t)\to\lrm{}(t+1)\to\dots $.
3471: Notice that, as the number of iterations increases, a given
3472: message can change its status according to one of the transitions
3473: $\?\to\g$, $\g\to\0$ or $\?\to\0$. Therefore the algorithms surely stops
3474: after a finite number of iterations (at most twice the number of edges
3475: in the graph). We shall denote the fixed point as $\lrm{}(\infty)$,
3476: $\rlm{}(\infty)$.
3477: At the $t^{\text{th}}$ iteration the algorithm deliver an estimate
3478: $\nu_i(t)$, $i\in[n]$ of the $i^{\text{th}}$ transmitted bit.
3479: %
3480: %**************************************************
3481: %
3482: \subsection{The Case of Tree Graphs and Some Simple Consequences}
3483: \label{sec:treecase}
3484: As for other message-passing algorithms, it is instructive
3485: to study the behavior of the \maxwell decoder on trees. In particular,
3486: we will show that:
3487: %(a) The \maxwell decoder completely solves
3488: %systems of parity check equations on trees;
3489: %(a) its analysis allows to
3490: %{\em count} the number of solutions of such linear systems in a {\em local} way;
3491: (a) On a tree the {\em sequential} \maxwell decoder guesses exactly as many
3492: variables as there are degrees of freedom in the system (implying that
3493: all these guesses are independent);
3494: (b) on a tree the number of {\em independent} guesses ventured
3495: by the (not necessarily sequential) \maxwell decoder
3496: by end of the decoding process is equal to the number of degrees
3497: of freedom of the system and it can be computed in a {\em local} way;
3498: (c) the same local counting formula gives in general (for Tanner graphs that
3499: are not necessarily trees) an {\em upper bound} on the number of independent guesses
3500: which remain at the end of the decoding process.
3501:
3502: %While these tasks are elementary, several of the ideas discussed
3503: %in this context can be carried over, with little modification, to
3504: %locally tree-like graphs.
3505:
3506: We have already %established that
3507: explained that, for the purpose of analysis, we can
3508: make the all-zero codeword assumption. Therefore, in the sequel
3509: we only have to consider linear systems of equations with a zero right side.
3510: We say that the \maxwell decoder is
3511: {\em bit-by-bit} (or {\em sequential}) if any time the BP phase comes to
3512: a halt, the decoder guesses a {\em single} unknown bit and then proceeds
3513: by processing all consequences until no further progress is achieved.
3514:
3515: \blemma[Number Of Guesses of Sequential \maxwell Decoder]
3516: \label{lem:numberofsequentialguesses}
3517: Consider a binary linear system of equations with right side equal to zero
3518: and $k$ degrees of freedom (i.e., $k$ is equal to the number of variables minus
3519: the rank of the system).
3520: Assume that the Tanner graph associated to this system is a tree.
3521: Then the sequential \maxwell decoder ventures exactly $k$ guesses during the
3522: decoding process and all these guesses are independent.
3523: \elemma
3524: \bproof
3525: Without loss of generality
3526: we can assume that there are no check leaf nodes.
3527: In fact, whenever degree-one check nodes are present,
3528: the standard BP decoder can be run until
3529: all such nodes have been removed.
3530: For each variable node which is removed in this fashion, the rank of the
3531: system is decreased by exactly one as well.
3532:
3533: We claim that the resulting system of equations has full rank.
3534: To see this, assume to the contrary
3535: that there is a non-zero linear combinations of
3536: equations that yields
3537: zero. Look at the Tanner graph corresponding to this subset of equations:
3538: all variable nodes have (even) degree at least two
3539: and all check nodes have degree at least two (as argued above).
3540: It is well known that
3541: a graph with minimum degree at least two contains at least one cycle,
3542: contradicting the hypothesis that the initial graph was a tree.
3543:
3544: Consider therefore a Tanner graph which is a tree and all
3545: of its leaf nodes are variables.
3546: Let $\ldegree_i$, $i \in [n]$, ($\rdegree_i$, $i \in [m]$)
3547: denote the degree of variable (check) node $i$.
3548: By our remarks above,
3549: the corresponding system of equations has $n-m$ degrees of freedom.
3550: Therefore, it is clear that the \maxwell decoder has to guess {\em at least}
3551: $n-m$ bits before it stops.
3552: We claim that it ventures exactly $n-m$ guesses, i.e.,
3553: that on a tree the sequential guesses are {\em independent}.
3554:
3555: At the start of the decoding process all messages are
3556: erasures. We will show that at the end of the decoding process
3557: each edge carries exactly one $\g$ message in one direction
3558: and a $\?$ message
3559: in the other direction. This proves our claim: it implies that
3560: a variable node which has been guessed, and hence all of its outgoing messages
3561: carry a $\g$ message, has no incoming $\g$ message.
3562: It is therefore not constrained
3563: by any of the other guesses, i.e., it is independent.
3564: Clearly, at the end of the decoding process
3565: each edge has to carry a $\g$ message in {\em at least one} direction;
3566: otherwise the connected bit has not been determined yet,
3567: contradiction the assumption that the \maxwell decoder has halted.
3568:
3569:
3570: % for search option: word natuerlich
3571: \begin{figure}[htp]
3572: \centering
3573: \setlength{\unitlength}{1bp}
3574: \begin{picture}(240,140)
3575: \put(5,5){\includegraphics[width=100bp]{treecheck}}
3576: \put(30,100){\begin{turn}{45}\makebox(0,0){{$\underset{\longrightarrow}{\g}$} }\end{turn}}
3577: \put(59,100){\begin{turn}{90}\makebox(0,0){{$\rightarrow$} }\end{turn}}
3578: \put(59,130){\begin{turn}{90}\makebox(0,0){{$\rightarrow$} }\end{turn}}
3579: \put(52,100){\begin{turn}{90}\makebox(0,0){{$\leftarrow$} }\end{turn}}
3580: \put(52,130){\begin{turn}{90}\makebox(0,0){{$\leftarrow$} }\end{turn}}
3581: \put(56,143){\makebox(0,0){$\edge$}}
3582: \put(56,85){\makebox(0,0){$\tilde{\edge}$}}
3583: \put(62,93){\makebox(0,0){$\g$}}
3584: \put(62,123){\framebox(10,10){${\tt \mathbf{g}}$}}
3585: \put(48,90){\makebox(0,0){$\g$}}
3586: \put(48,120){\makebox(0,0){$\g$}}
3587: \put(82,98){\begin{turn}{-45}\makebox(0,0){{$\underset{\longleftarrow}{\g}$} }\end{turn}}
3588: \put(130,0)
3589: {
3590: %\put(30,100){\begin{turn}{45}\makebox(0,0){{$\underset{\longrightarrow}{\?}$} }\end{turn}}
3591: \put(56,143){\makebox(0,0){$\edge$}}
3592: \put(56,85){\makebox(0,0){$\tilde{\edge}$}}
3593: \put(60,100){\begin{turn}{90}\makebox(0,0){{$\rightarrow$} }\end{turn}}
3594: \put(60,130){\begin{turn}{90}\makebox(0,0){{$\rightarrow$} }\end{turn}}
3595: \put(51,100){\begin{turn}{90}\makebox(0,0){{$\leftarrow$} }\end{turn}}
3596: \put(51,130){\begin{turn}{90}\makebox(0,0){{$\leftarrow$} }\end{turn}}
3597: \put(62,93){\makebox(0,0){$\g$}}
3598: \put(62,123){\framebox(10,10){${\tt \mathbf{g}}$}}
3599: \put(48,90){\makebox(0,0){$\g$}}
3600: \put(48,120){\makebox(0,0){$\g$}}
3601: %\put(82,98){\begin{turn}{-45}\makebox(0,0){{$\underset{\longleftarrow}{\?}$} }\end{turn}}
3602: }
3603: \put(135,5){\includegraphics[width=100bp]{treevar}}
3604: \put(55,-4){\makebox(0,0){\small{(a)}}}
3605: \put(190,-4){\makebox(0,0){\small{(b)}}}
3606: \put(63,12){\makebox(0,0){$\g$}}
3607: \put(193,12){\makebox(0,0){$\g$}}
3608: \end{picture}
3609: \caption{
3610: In (a) consider the messages flowing along edge $\edge$. Assume that
3611: the {\em outgoing} message (shown in a frame) switches as a consequence of a newly
3612: guessed bit from $\?$ to $\g$. Assume further that the {\em incoming}
3613: message flowing in the opposite direction
3614: is $\g$ as well. This provides the induction step from odd levels to even levels.
3615: As indicated in the figure, it then follows that both messages
3616: along edge $\tilde{\edge}$ are $\g$ as well. The case of an edge exiting a variable
3617: node is shown in (b) and follows by essentially the same argument.
3618: }
3619: \label{fig:independentproof}
3620: \end{figure}
3621:
3622: Let us show that it can not carry a $\g$ message in both directions.
3623: Initially all messages are $\?$. The sequential \maxwell decoder
3624: proceeds in phases, guessing a bit and then determining all
3625: consequences of this guess during the BP phase until it gets stuck again.
3626: Let us call one such guess followed by the BP phase one {\em iteration}.
3627: Let us agree that during the BP phase the consequence of a newly guessed bit
3628: are computed {\em in order of increasing distance} from the guessed bit. This means,
3629: that we first process all edges directly connected to this bit
3630: (call this {\em level zero}), then all edges at distance one (call this {\em level one})
3631: and so on. Assume that when we process level $\iter$, $\iter \geq 1$,
3632: we encounter an edge whose
3633: outgoing (away from the newly guessed bit) message switches from $\?$ to $\g$
3634: and whose incoming message already is $\g$. We claim that then the same
3635: must have occurred at level $\iter-1$. This is quickly verified by checking
3636: explicitly both cases:
3637: an edge which goes from a check node to a variable node
3638: (odd levels $\iter$; left picture in Fig.~\ref{fig:independentproof})
3639: and the case of an edge which
3640: goes from a variable node to a check node
3641: (even levels $\iter$; left picture in Fig.~\ref{fig:independentproof}).
3642: If we apply this argument inductively, we see that the guessed variable node
3643: must have had an incoming message which was $\g$, contradicting the fact
3644: that the \maxwell decoder decided to guess this bit.
3645: \eproof
3646:
3647: What happens if we run the \maxwell decoder in a non-sequential way, i.e.,
3648: if we guess many/several bits each time we get stuck? In this
3649: case it can happen that some of the guesses are dependent.
3650: Nevertheless, the number of independent guesses remaining at the end
3651: of the process is
3652: still equal to the degrees of freedom of the system of equations.
3653: More importantly, on a tree this number of independent guesses
3654: can be computed in a {\em local} way.
3655: \blemma[Number of Independent Guesses]
3656: \label{lem:independentguesses}
3657: Consider a binary linear system of equations with right side equal to zero
3658: and $k$ degrees of freedom (i.e., $k$ is equal to the number of variables minus
3659: the rank of the system).
3660: Assume that the Tanner graph associated to this system is a tree
3661: and that it contains no check nodes of degree one.
3662: Then the number of {\em independent} guesses ventured by the \maxwell decoder
3663: at the end of the decoding process is equal to $k$.
3664: Further, let ${\mathbb{G}}$ denote the total number of guesses of the \maxwell decoder,
3665: denote by $\ldegree^{\g}_{i}$ the number of
3666: incoming $\g$ messages at variable node $i$ (including, if applicable, the guess
3667: of the bit itself), and by ${\cal C}_{\g}$ the subset of
3668: all check nodes all of its incoming messages are $\g$. Then
3669: \begin{align}
3670: \label{equ:countingontreeone}
3671: k & = {\mathbb{G}}-
3672: \sum_{i \in {\cal V}} (\ldegree^{\g}_i-1)+
3673: \sum_{i \in {\cal C}_{\g}} (\rdegree_i-1) .
3674: \end{align}
3675: \elemma
3676: \bproof
3677: By definition of the algorithm, at the end of the decoding
3678: process all bits have been determined (i.e., guessed or expressed
3679: in terms of guessed bits). This means that among the guesses ventured
3680: by the \maxwell decoder there must be $k$ independent such guesses.
3681: Now note that the final state of the messages is independent of the
3682: order in which the guesses are taken. It is convenient to imagine
3683: that we first venture the $k$ independent guesses and then apply the BP decoder.
3684: At the end of this phase all bits are known. Further, from
3685: Lemma \ref{lem:numberofsequentialguesses}
3686: we know that $\ldegree^{\g}_i=1$ for all $i \in [n]$
3687: and ${\cal C}_{g}$ is the empty set.
3688: Therefore, the stated counting formula is correct at this stage.
3689: Assume now we proceed in iterations, adding one guess at a time
3690: and propagating all its consequences. We will verify that
3691: the counting formula stays valid.
3692: Assume therefore that the counting formula is
3693: correct at the start of an iteration and add a further guess, lets say of variable $i$.
3694: This extra guess increases $\ldegree^{\g}_i$ by one and increases the
3695: number of guesses by one, keeping the counting formula intact.
3696: Consider now the ensuing BP phase. Consider an edge $\edge$ emanating from
3697: a variable node $i$, the check node connected to it, call it $j$
3698: and all the edges and variable nodes connected to this check node.
3699: Assume that the message from $i$ to $j$ is $\?$ (in the case
3700: that this message is already $\g$, the message does not change
3701: and there is nothing to prove).
3702: As a consequence the
3703: message from $j$ to $i$ must be a $\g$ because of the argument above.
3704: Also, all the incoming messages into $j$ but the one form $i$
3705: must be $\g$ as well (otherwise the update rule would have been violated
3706: at node $j$).
3707: Update all the corresponding
3708: edge messages. If the message from $i$ to $j$ does not change,
3709: then neither does any of the messages outgoing at the check node
3710: and the counting formula stays valid. If, on the other hand, the outgoing
3711: message along edge $\edge$ flips to $\g$ then
3712: so do all the messages outgoing from the check node $j$.
3713: Assume that the check node has degree $\rdegree_j$.
3714: Then, ${\cal C}_{g}$ now contains $j$. This increases the right hand side
3715: of the counting formula by $\rdegree_j-1$. On the other hand it also increases
3716: $\ldegree^{\g}_l$ by one for all $l \in {\cal V}$ which are connected to check node $j$, but for node $i$ (the corresponding message was already a $\g$).
3717: In total this decreases the right hand side of the counting formula by $\rdegree_j-1$.
3718: \eproof
3719:
3720: Each part of the counting equation (\ref{equ:countingontreeone}) has a pleasing
3721: interpretation. As stated, ${\mathbb{G}}$ is the total number of ventured guesses.
3722: If a variable node has $\ldegree^{\g}$ incoming $\g$ messages then
3723: these correspond to $\ldegree^{\g}$ linear equations, each of which
3724: determines the same bit. This gives rise to $(\ldegree^{\g}-1)$ linear
3725: conditions which the ${\mathbb{G}}$ guesses have to fulfill. But not all these conditions
3726: are linearly independent.
3727: Consider Fig.~\ref{fig:correctionterm}. If a check node of degree $\rdegree$
3728: has all of its incoming messages equal to $\g$ then the $\rdegree$ equations
3729: which correspond to the $\rdegree$ outgoing messages are identical, i.e.,
3730: $\rdegree-1$ of them are linearly dependent. The last term in the counting
3731: formula (\ref{equ:countingontreeone}) therefore corrects the over-counting
3732: of dependent conditions.
3733: \begin{figure}[hbt]
3734: \vspace{5bp}
3735: \centering
3736: \setlength{\unitlength}{0.5bp}
3737: \begin{picture}(360,180)
3738: \put(0,20){\includegraphics[scale=0.5]{correctionterm}}
3739: \put(200,20){\includegraphics[scale=0.5]{correctionterm}}
3740: \put(155,100){\makebox(0,0){$\g_3$}}
3741: \put(15,155){\makebox(0,0){$\g_1$}}
3742: \put(15,40){\makebox(0,0){$\g_2$}}
3743: \put(251,91){\begin{turn}{45}\makebox(0,0){{$\underset{\longrightarrow}{\g_2}$} }\end{turn}}
3744: \put(269,69){\begin{turn}{45}\makebox(0,0){{$\overset{\longleftarrow}{\g}$} }\end{turn}}
3745: \put(266,133){\begin{turn}{-45}\makebox(0,0){{$\underset{\longrightarrow}{\g_1}$} }\end{turn}}
3746: \put(248,112){\begin{turn}{-45}\makebox(0,0){{$\overset{\longleftarrow}{{\g}}$} }\end{turn}}
3747: \put(310,114){\begin{turn}{0}\makebox(0,0){{$\underset{\longrightarrow}{\g}$} }\end{turn}}
3748: \put(310,86){\begin{turn}{0}\makebox(0,0){{$\overset{\longleftarrow}{\g_3}$} }\end{turn}}
3749: \put(50,0){\makebox(0,0){\footnotesize{time $t$} }}
3750: \put(250,0){\makebox(0,0){\footnotesize{time $t+1$}}}
3751: \end{picture}
3752: \caption{Computation of the number of linearly
3753: independent conditions. To each of the incoming edges corresponds a list.
3754: To keep things simple and without essential loss of generality, assume that
3755: $\List_i=\{i\}$. The three outgoing lists are then $\List_1=\{2, 3\}$,
3756: $\List_2=\{1, 3\}$, and $\List_3=\{1, 2\}$. Compare the incoming and
3757: outgoing list at node 1: we get the condition $x_1=x_2+x_3$. But
3758: exactly the same condition appears at node $2$ and node $3$. In general, a
3759: check node of degree $\rdegree$, all of its incoming messages are $g$, generates
3760: $\rdegree-1$ linearly dependent conditions.}
3761: \label{fig:correctionterm}
3762: \end{figure}
3763:
3764: \bex
3765: Consider a code whose Tanner graph is a
3766: tree and all leaves are variable nodes. Let the set
3767: of variables (checks) be indexed by $[n]$ ($[m]$, and let
3768: $\ldegree_i$, $i \in [n]$, ($\rdegree_i$, $i \in [m]$)
3769: be the degree of variable (check) node $i$. Assume that the \maxwell
3770: decoder guesses all leaf (variable) nodes and then proceeds by
3771: message passing. It is not very hard to see that in this setting the decoder
3772: proceeds with the message-passing phase (starting from the leaf nodes) until all variables have
3773: been determined and that no further guesses have to be made. Further, at the end of the decoding
3774: process {\em all} messages are $\g$.
3775:
3776: Let us determine the number of independent guesses at
3777: the end of the decoding process using the counting formula
3778: (\ref{equ:countingontree}).
3779: Note that for each leaf node we have $\ldegree^{\g}=2$ (one guess and
3780: one additional incoming $\g$ message. For all internal variable nodes
3781: we have $\ldegree^{\g}=\ldegree$. Finally, ${\cal C}_{\g}={\cal C}$.
3782: If we let $n_l$ denote the number leaf nodes, so that ${\mathbb{G}}=n_l$, we get
3783: that the number of independent guesses is equal to
3784: \begin{align*}
3785: & n_l - \sum_{i \in \text{leaves} } (2-1) -
3786: \sum_{i \in [n] \setminus \text{leaves} } (\ldegree_i-1) +
3787: \sum_{i \in [m]} (\rdegree_i-1) \\
3788: = &
3789: -\sum_{i \in [n] } (\ldegree_i-1) +
3790: \sum_{i \in [m]} (\rdegree_i-1) = n-m.
3791: \end{align*}
3792: This is of course the expected result since the
3793: system has exactly $n-m$ degrees of freedom.
3794: \eex
3795: %If the number of edges is denoted by $e$, we have the identities
3796: %$e=n+m-1$, $\sum_{i \in [n]} \ldegree_i=\sum_{i \in [m]} \rdegree_i = e$. We
3797: %assume $\rdegree_i \geq 2$ and we say that a variable node $i$ is a {\em leaf}
3798: %if $\ldegree_i=1$.
3799: %Finally, assume for the sake of simplicity that the
3800: %transmitted message has been completely erased and that the leaves
3801: %have been guessed. In other words $\m^{\epsilon}_i = \g$ if
3802: %$i$ is a leaf, and $\m^{\epsilon}_i = \?$ otherwise.
3803: %
3804: %In such a setting the \maxwell decoder converges, after a finite
3805: %number of iterations, to the fixed point $\lrm{}(\infty)=\rlm{}(\infty) =\g$
3806: %for every edge and $\nu_i(\infty) =\g$ for every vertex. Moreover,
3807: %the corresponding resolution lists are non-empty for any of the edges
3808: %(vertices). In fact, consider the message $\m_i$ and
3809: %the corresponding list $\List_i$ flowing
3810: %along edge $i$ in a particular direction.
3811: %It is clear that $\List_i$ can only contain indices of leaves lying
3812: %``in the past'' of this edge (i.e., in the subtree which we obtain if we expand out the
3813: %edge in opposite direction). Consider now a check node update. By the tree property,
3814: %the subtrees corresponding to the various incoming edges must be distinct, so
3815: %that the incoming lists are disjoint and no cancellations can happen.
3816: %
3817: %The algorithm also produces a list of conditions. A variable leaf node
3818: %produces exactly one condition, since it has the received value (guessed value)
3819: %which must be equal to the incoming message.
3820: %If $i$ is an internal (i.e.
3821: %non-leaf) variable node, then $|{\cal L}_\g| = \ldegree_i$, since by
3822: %%assumption the received message has been erased and there are $\ldegree_i$
3823: %incoming messages all of which are $\g$. Therefore, for variable node $i$ (leaf or
3824: %internal) $\ldegree_i-1$ conditions are produced.
3825: %The total number of conditions is hence $\sum_{i \in [n]} (\ldegree_i-1)$.
3826: %%Any solution (codeword) of the linear system on the tree
3827: %%can be constructed by assigning to the leaves values which
3828: %%satisfy such conditions and using the resolution lists $\List_i$ do determine
3829: %%internal variables. Formally this can be shown
3830: %by induction on the number of levels of the tree.
3831: %\end{example}
3832:
3833: %Let $2^{\entropy}$ denote the number of solutions. Since the $m$ parity check nodes are
3834: %linearly independent (this follows from the tree property), we have $\entropy=n-m$.
3835: %However, we would like to compute $\entropy$ by analyzing the \maxwell algorithm.
3836: %Notice that there is a one-to-one correspondence between solutions
3837: %and assignments to the guessed variables which satisfy the $\sum(\ldegree_i-1)$
3838: %conditions. Therefore $\entropy=n_{\rm l}-n_{\rm c}$, where $n_{\rm l}$ denotes
3839: %the number of leaf nodes and $n_{\rm c}$
3840: %is the number of {\em linearly independent} conditions produced by the
3841: %algorithm. Of course $n_{\rm c} \le \sum(\ldegree_i-1)$.
3842: %Furthermore, it is easy to realize that $n_{\rm c}$ is strictly less than
3843: %the total number of conditions produced.
3844: %For each internal (i.e. non adjacent to leaves)
3845: %parity-check node $i$, the $\rdegree_i$ conditions
3846: %arising at neighboring variable nodes
3847: %and involving lists coming from $i$ are dependent. In fact, they all
3848: %coincide. As a consequence $n_{\rm c}$ is not larger than
3849: %$\sum_{i \in [n]} (\ldegree_i-1)-\sum_{i \in [m]'}(\rdegree_i-1)$, where the second
3850: %sum only extends over internal check nodes.
3851: %One can prove (in the tree case)
3852: %that the remaining conditions are linearly independent.
3853: %We claim that this is exact. Let us check this explicitly.
3854: %%
3855: %\begin{eqnarray*}
3856: %%
3857: %\entropy & = &
3858: %n_{\rm l} - n_{\rm c} =\\
3859: %& = & n_{\rm l} -\sum (\ldegree_i-1)+\sum{}'\, (\rdegree_a-1)=\\
3860: %& = & n_{\rm l} -\sum (\ldegree_i-1)+\sum(\rdegree_a-1) -\sum_{{\rm leaves}}
3861: %(\rdegree_a-1)=\\
3862: %& = & n_{\rm l} - (e-n)+(e-m) -n_{\rm l} =\\
3863: %& = & n-m\, ,
3864: %%
3865: %\end{eqnarray*}
3866: %
3867: %which is the correct result.
3868:
3869: So far we have only considered sets of equations whose Tanner graph is a
3870: tree. What happens if we run the \maxwell decoder on a general system of
3871: equations. For a general Tanner graph, the above counting of the total
3872: number of independent guesses is not necessarily tight. The counting
3873: of the total number of conditions generated by the \maxwell decoder is
3874: always correct. But it can happen that besides the obvious over-counting
3875: at check nodes, there are other dependencies generated by loops in the
3876: graph which are not considered in the counting formula. Therefore,
3877: in general we only get a lower bound. Let us state this explicitly.
3878: %
3879: \begin{lemma}[Lower Bound on Independent Guesses]
3880: \label{LemmaConditions}
3881: Consider a binary linear system of equations with right side equal to zero
3882: and $k$ degrees of freedom (i.e., $k$ is equal to the number of variables minus
3883: the rank of the system).
3884: Assume that the Tanner graph associated to this system
3885: contains no check nodes of degree one.
3886: Let ${\mathbb{G}}$ denote the number of all guesses of the \maxwell decoder, denote
3887: by $\ldegree^{\g}_{i}$ the number of
3888: incoming $\g$ messages at variable node $i$ (including the guess
3889: if this node has been guessed), and by ${\cal C}_{\g}$ the subset
3890: of all check nodes all of whose incoming messages are $\g$. Then
3891: \begin{align}
3892: \label{equ:countingontree}
3893: k & \geq {\mathbb{G}}-
3894: \sum_{i \in {\cal V}} (\ldegree^{\g}_i-1)+
3895: \sum_{i \in {\cal C}_{\g}} (\rdegree_i-1) .
3896: \end{align}
3897: %
3898: %Consider the application of a \maxwell decoder to a low density parity check
3899: %code with degrees $\ldegree_i$ (variables) and $\rdegree_i$
3900: %(checks) and assume that $\m^{\epsilon}_i\in\{\g,\?\}$ for any $i\in[n]$.
3901: %Let $n_{\rm c}$ denote the number of unconfirmed guesses
3902: %ventured by the algorithm at the end of the decoding process. Denote by ${\cal L}_{i,\g}$ the set of
3903: %incoming $\g$ messages at variable node $i$, and by ${\cal C}$ the set of
3904: %check nodes whose outgoing messages are only $\g$'s.
3905: %Then
3906: %%
3907: %\begin{eqnarray}
3908: %%
3909: %n_{\rm c}\le \sum_{i}\max(|{\cal L}_{i,\g}|-1,0)-\sum_{a\in {\cal C}}
3910: %(\rdegree_a-1)\, .\label{UBIndependent}
3911: %%
3912: %\end{eqnarray}
3913: %
3914: %
3915: \end{lemma}
3916:
3917: %
3918: %**************************************************
3919: %
3920: %\subsection{A First Handwaving Analysis}
3921: %Before we venture into a more careful analysis of the performance
3922: %of the \maxwell decoder let us give here a quick outline.
3923: %First, in order to avoid unnecessary complications
3924: %we restrict ourselves here to degree distribution pairs
3925: %that have a single jump (BP and MAP). The general case can
3926: %be dealt with in a similar manner by stringing together several
3927: %of the decoding phases discussed below.
3928: %
3929: %We can imagine that we separate the workings of the \maxwell decoder
3930: %into two phases. The first phase consists
3931: %of the guessing phase, whereas in the second phase
3932: %we only have confirmations. This separation can
3933: %be accomplished as follows:
3934: %Assume the channel parameter is $\epsilon$ with $\epsilon > \epsilon^{\MAP}$.
3935: %Imagine a very long graph instance and assume we pick $\gamma$
3936: %in such a way that $\epsilon^{\BP} < \epsilon (1-\gamma) < \epsilon^{\BP}+\delta$,
3937: %where $\delta$ is strictly positive but very small.
3938: %
3939: %Imagine now that we run a (sequential) Maxwell decoder on this input.
3940: %Because of our choice of parameters, the Maxwell decoder
3941: %terminate after a finite number of iterations and
3942: %the degree distribution of the residual graph is close
3943: %to the degree distribution at the critical point. Essentially
3944: %the same conclusion holds if we run the Maxwell decoder
3945: %only for a large but finite number of iterations.
3946: %We are therefore in the realm of decoding on a tree, a situation which
3947: %we discussed in detail in the previous section.
3948: %We have seen that on a tree there are no contradictions
3949: %and that the number of {\em useful} guesses is equal to ...
3950: %This concludes the first phase.
3951: %
3952: %Consider now the second phase in some more detail.
3953: %More precisely we proceed as follows. Consider the residual graph
3954: %right at the critical point. This graph corresponds to a set of
3955: %linear equations. In general this system of equations has
3956: %a non-zero right side, and this right side is a function of
3957: %all the guesses that have been ventured so far. Consider the same
3958: %set of linear equations but with a zero right side. Run the BP
3959: %decoder on this graph. By construction this BP decoder finishes
3960: %with high probability. How many confirmations are created
3961: %in the process of running the BP decoder. This number is exactly
3962: %equal to the difference between the number of variable nodes
3963: %in the residual graph and the number of checks. Some calculus
3964: %(THIS HAS TO BE CONNECTED PROPERLY) shows that this difference
3965: %is exactly proportional to the dark gray area under EBP \exitentropy curve.
3966: %
3967:
3968:
3969: %
3970: %**************************************************
3971: %
3972: \subsection{Density Evolution Analysis}
3973: \label{sec:densityevolution}
3974:
3975: Let us now perform the usual DE analysis.
3976: Let $\lra^t_{\lrm{}}$ denote the probability that a left-to-right message at time $t$
3977: is equal to $\lrm{}\in\{\0,\?,\g\}$, and let
3978: $\rla^t_{\rlm{}}$ denote the corresponding probability for a
3979: right-to-left message.
3980:
3981: (i) At the check node side the DE relations read
3982: \begin{align*}
3983: \rla_{\0}^{t} & =\redge(\lra_{\0}^{t}),\\
3984: \rla_{\?}^{t} & = 1-\redge( \lra_{\0}^{t}+ \lra_{\g}^{t} )=1-\redge(1-\lra^t_{\?}),\\
3985: \rla_{\g}^{t} & =1-\rla_{\0}^{t}-\rla_{\?}^{t}=\redge(\lra_{\0}^{t}+ \lra_{\g}^{t})- \redge(\lra_{\0}^{t}).
3986: \end{align*}
3987:
3988: (ii) At the variable node side the DE relations are
3989: \begin{align*}
3990: \lra_{\0}^{t+1} & = 1-\ih \ledge( \rla_{\g}^t + \rla_{\?}^t ),\\
3991: \lra_{\?}^{t+1} & = (1-\gamma) \ih \ledge(\rla_{\?}^t),\\
3992: \lra_{\g}^{t+1} & = \ih \ledge( \rla_{\g}^t + \rla_{\?}^t ) - (1-\gamma)\ih\ledge(\rla_{\?}^t).
3993: \end{align*}
3994: According to our convention, the iteration counter is
3995: increased only in the variable node operation. Moreover,
3996: the variables $\lra_{\?}^{t}$ ($\rla_{\?}^{t}$) and
3997: $\lra_{\?}^{t}+\lra_{\g}^{t}$ ($\rla_{\?}^{t}+\rla_{\g}^{t}$)
3998: satisfy the same equations as the fractions of erased messages
3999: in the standard BP decoder with erasure probabilities
4000: $\cp(1-\gamma)$ and $\gamma$, respectively. This is an immediate consequence of the
4001: update rules defined in section \ref{MessagePassingSection}.
4002:
4003: When the time $t$ tends to $\infty$, DE converges to the fixed-point
4004: probability distribution.
4005: To settle our notation, we write
4006: $(\lra_{\0}^{t},\lra_{\?}^{t},\lra_{\g}^{t})\underset{t\to\infty}{\longrightarrow}\left(\lra_{\0}^{\infty}(\ih,\gp),\lra_{\?}^{\infty}(\ih,\gp) ,\lra_{\g}^{\infty}(\ih,\gp)\right)$
4007: and equivalently
4008: $(\rla_{\0}^{t},\rla_{\?}^{t},\rla_{\g}^{t})\underset{t\to\infty}{\longrightarrow}\left(\rla_{\0}^{\infty}(\ih,\gp),\rla_{\?}^{\infty}(\ih,\gp) ,\rla_{\g}^{\infty}(\ih,\gp)\right)$ .
4009: Observe that $\lra_{\?}^{\infty}(\ih,\gp)$ satisfies the equation $\lra = \ih(1-\gp)\ledge(1-\redge(1-\lra))$,
4010: while $\lra_{\0}^{\infty}(\ih,\gp)=\lra_{\0}^{\infty}(\ih)$
4011: satisfies the equation $(1-\lra) = \ih \ledge(1-\redge(1-(1-\lra)))$.
4012:
4013: Notice that the asymptotic state of the algorithm has the following structure.
4014: The variable nodes such that $\nu_i(\infty)=\?$ {\em or} $\nu_i(\infty) = \g$,
4015: form a stopping set: in fact this is the largest stopping set contained in the
4016: set of variable nodes for which $\mu_i^{\cp} = \?$ {\em or}
4017: $\mu_i^{\cp} = \g$. Further, the set of variable nodes such that
4018: $\nu_i(\infty)=\?$ form a stopping set contained in the previous one:
4019: this is the largest stopping set contained in the set $\mu_i^\cp = \?$.
4020:
4021: In the analysis below we shall repeatedly use the following trick.
4022: We shall compute expectations with respect to asymptotic ($t=\infty$)
4023: incoming messages in a given node. In such computations,
4024: we shall treat such messages as i.i.d. with distribution
4025: $\left(\lra_{\0}^{\infty},\lra_{\?}^{\infty},\lra_{\g}^{\infty} \right)$,
4026: (for left-to-right messages) or
4027: $\left(\rla_{\0}^{\infty},\rla_{\?}^{\infty},\rla_{\g}^{\infty} \right)$,
4028: (for right-to-left messages).
4029: As long as $(\cp,\gp)$
4030: take non-exceptional values, i.e., at continuity points of
4031: $\left(\lra_{\0}^{\infty}(\ih,\gp),\lra_{\?}^{\infty}(\ih,\gp),
4032: \lra_{\g}^{\infty}(\ih,\gp) \right)$, cf. Section \ref{sec:Counting},
4033: this is justified as follows. First consider messages after a finite
4034: number of iterations $t$. For $n$ large enough these are independent because
4035: the Tanner graph is locally a tree. But, if $(\cp,\gamma)$
4036: is non-exceptional the number of message which change between the
4037: $t^{\text{th}}$ iteration and the asymptotic state is bounded by $n\delta(t)$
4038: with $\delta(t)\to 0$ as $t\to\infty$. This argument is essentially the same
4039: as the one of App.~\ref{ProofCountingDegree}.
4040: %
4041: %************************************************************
4042: %
4043: \subsection{Guessing Strategy}
4044: \label{sec:guessstrat}
4045:
4046: In the analysis of the \maxwell decoder, we can chose the order
4047: of guesses at our convenience. As long as the message is completely
4048: decoded and the final estimates are
4049: $\nu_i(\infty)\in\{\0,\g\}$ for any bit $i$,
4050: the algorithm realizes a complete list decoding.
4051:
4052: We shall adopt the following strategy: we perform $n_{\text{rounds}}$
4053: ``decoding rounds''. Our progress will be measured by
4054: the parameter $\gamma$, which is initially set to zero
4055: and which advances by $\Delta \gamma = 1/n_{\text{rounds}}$
4056: in each round.
4057:
4058: Set $\gamma=0$. Start with the messages received via
4059: BEC$(\ih)$ and apply BP decoding until the algorithm gets stuck.
4060: Then consider each of the bits not yet determined
4061: and set $\mu_i^{\ih} = \g$ independently for each of them with probability
4062: $\Delta\gamma /(1-\gamma)$. (In the first round this probability
4063: is equal to $\Delta \gamma$.)
4064: Set $\gamma\defas\gamma+\Delta\gamma$.\footnote{Note
4065: that if a bit is first selected with probability $\gamma$
4066: and then independently selected with probability $\Delta \gamma/(1-\gamma)$,
4067: then the probability that it was selected at least once is equal to $\gamma+\Delta \gamma$.
4068: This is the rational for our choice of parameters.}
4069: Apply the \maxwell decoder until it
4070: gets stuck.
4071: This is repeated $n_{\text{rounds}}$ times until $\gamma=1$.
4072: If at any earlier phase complete decoding is achieved, the algorithm
4073: is halted and the current set of decoded codewords output.
4074:
4075: The analysis becomes simpler (and the algorithm more efficient)
4076: if we take $\Delta\gamma\to 0$. We shall always think of this limit being taken
4077: after $n\to\infty$. We will see that in this limit the appearance of contradictions
4078: is sharply concentrated to those rounds which include a discontinuity of
4079: the EXIT curve. In other words, we will see that the algorithm
4080: alternates between the following two phases which are
4081: well separated: in the ``guessing phase'' the algorithm guesses
4082: a small fraction of bits and the processes the
4083: consequences but theses consequences do not propagate too far
4084: and essentially stay local; in the ``contradiction phase'' on
4085: the other hand the algorithm suddenly
4086: discovers many relationships (finds many contradictions) and
4087: the size of the residual graph changes by a constant fraction
4088: which is independent of the step size $\Delta \gamma$.
4089: %
4090: %************************************************************
4091: %
4092: \subsection{Analysis: Guess Work}
4093: \label{sec:guesswork}
4094:
4095: %We start by computing the number of newly guessed variables when $\gamma$
4096: %is changed by an amount $\Delta\gamma>0$.
4097: Consider a non-exceptional point $(\cp,\gp)$ and let
4098: $\n\Delta {\mathbb{G}}$ be the number of newly guessed variables when $\gamma$
4099: is changed by an amount $\Delta\gamma>0$.
4100: %How many of the new guesses are linearly independent?
4101: %This question is related to our investigation in Section \ref{sec:treecase} of how
4102: %the \maxwell decoder proceeds if we run it in a non-sequential manner
4103: %and guess several bits at each time.
4104:
4105: The process can be described as follows. For each $i\in[n]$, $i$
4106: is selected independently with probability $\Delta\gamma/(1-\gamma)$.
4107: For each selected bit, we consider the present estimate provided by
4108: the \maxwell decoder: $\nu_i(\infty)\in\{\0,\g,\?\}$.
4109: If $\nu_i(\infty) = \?$, the observation on $i$ is changed
4110: from $\m^\cp_i=\?$ to $\m^\cp_i = \g$: the counter of newly guessed variables is
4111: increased by one.
4112: By linearity of expectation, we get
4113: %
4114: \begin{align*}
4115: %
4116: \expectation [
4117: \Delta{\mathbb{G}}] & = \frac{1}{\n}\sum_{i\in [n]}\pr(\mbox{$i$ is selected})\,
4118: \pr(\nu_i(\infty) = \?) \\
4119: & = \frac{\Delta\gamma}{1-\gp}\,
4120: \cp (1-\gp) \lnode(\rla_{\?}^{\infty})
4121: = \cp \lnode(\rla_{\?}^{\infty})\Delta\gamma\, .
4122: %
4123: \end{align*}
4124: %
4125: Notice that, in this computation we assumed $n\to\infty$ and $t\to\infty$
4126: afterwards.
4127:
4128: Recall that, after $\gamma$ is changed to $\gamma+\Delta\gamma$
4129: and the $n \Delta{\mathbb{G}}$ new guesses are introduced, the message
4130: passing \maxwell decoder is started again until a new fixed point is reached.
4131: %
4132: %***************************************************************
4133: %
4134: \subsection{Analysis: Confirmation Work}
4135:
4136: At each step of the above algorithm, it may happen that several
4137: $\g$ messages are transmitted to the same variable node $x_i$.
4138: Each of these lists corresponds to
4139: a distinct resolution rule for $x_i$.
4140: Their convergence on the same node imposes some non-trivial
4141: condition on the variables which appear in the resolution rules.
4142: Here we estimate the number of independent such conditions by
4143: exploiting Lemma~\ref{LemmaConditions} above.
4144: Notice that in Lemma~\ref{LemmaConditions} we assume $\m^{\cp}_i\in\{\g,\?\}$.
4145: In order to make contact with this assumption we could
4146: first run the classical BP decoder until
4147: no further progress can be made.
4148: We could now directly apply Lemma~\ref{LemmaConditions}
4149: to the {\em residual} graph. The disadvantage of this strategy
4150: is that in this scheme it is not so straightforward to
4151: relate the progress of the \maxwell decoder on the residual
4152: graph to the original DE equations.
4153:
4154: Alternatively we can apply Lemma~\ref{LemmaConditions} directly
4155: to the original graph if (i) we do not count contradictions generated
4156: at variable nodes which receive at least one $0$ message (either from
4157: the channel or from the graph) and (ii) we count towards the degree of a check
4158: node only those edges whose incoming messages are not $0$.
4159: With these two conventions one can check that Lemma~\ref{LemmaConditions}
4160: holds for a general graph including degree-one check nodes as well
4161: as variable nodes which are known.
4162:
4163: Let $(\cp,\gp)$ be a non-exceptional point and denote by $n{\mathbb C}$
4164: the number of contradictions as estimated by the right-hand side of
4165: (\ref{equ:countingontree}). The first term counts
4166: the number of conditions arising at that node.
4167: We get
4168: %
4169: \begin{multline*}
4170: %
4171: \expectation\left\{\frac{1}{n}\sum_{i\in\cal V}\max(|{\cal L}_{i,\g}|-1,0)
4172: \right\} =\\
4173: \cp(1-\gp)\sum_{\ldegree}\lnode_{\ldegree}
4174: \expectation_{\ldegree}\left\{\max(n_\g-1,0)\,\ind_{n_\0 = 0}\right\} \\
4175: +\cp\gp \sum_{\ldegree}\lnode_{\ldegree}
4176: \expectation_{\ldegree}\left\{\max(n_\g,0)\,\ind_{n_\0 = 0}\right\} \, ,
4177: %
4178: \end{multline*}
4179: %
4180: where $\ind_A$ is the indicator function for the event $A$ and
4181: where $n_\g$, $n_\0$, and $n_\?$ count the number of incoming $\g$, $\0$,
4182: and $?$ messages.
4183: Here the limits $n\to\infty$ and $t\to\infty$ are understood and
4184: $\expectation_{\ldegree}$ denotes expectation with respect the multinomial
4185: variables $n_\0,n_\g,n_\?$ with sum $\ldegree$ and parameters
4186: $\rla_{\0}^{\infty},\rla_{\*}^{\infty},\rla_{\?}^{\infty}$.
4187: Note that we have the indicator function $\ind_{n_\0 = 0}$
4188: since by our remarks above we should only consider nodes
4189: ``in the residual graph'', i.e., nodes which were not
4190: already determined in the BP phase as a consequence of the received
4191: bits.
4192: Throughout this section we shall adopt
4193: the shorthands $\rla_{\0},\rla_{\*},\rla_{\?}$ for
4194: $\rla_{\0}^{\infty},\rla_{\g}^{\infty},\rla_{\?}^{\infty}$
4195: (and analogous ones for left-to-right messages).
4196: By computing these expectations we get
4197: %
4198: \begin{multline}
4199: %
4200: \expectation\left\{\frac{1}{n}\sum_{i\in\cal V}\max(|{\cal L}_{i,\g}|-1,0)
4201: \right\} =\\
4202: \cp(1-\gp)\left\{\lnode'(\rla_\?+\rla_\g)\rla_\g -\lnode(\rla_\?+\rla_\g)
4203: +\lnode(\rla_\?)\right\} \\
4204: +\cp\gp \lnode'(\rla_\?+\rla_\g)\rla_\g\, .\label{Cond1}
4205: %
4206: \end{multline}
4207: %
4208: We must now evaluate the correction term in (\ref{equ:countingontree}).
4209: Consider a check node $a$. Assume that its ``residual'' degree is $\rdegree'_a$. I.e.,
4210: $\rdegree'_a$ counts the number of edges whose incoming messages are not zero.
4211: If the corresponding $\rdegree{}'_a$ outgoing messages
4212: are all $\g$ (equivalently, the $\rdegree'_a$ ingoing messages
4213: are all $\g$), then the same condition has been overcounted $\rdegree{}'_a-1$
4214: times. We denote the set of such check nodes as ${\cal C}$ and obtain
4215: %
4216: \begin{multline*}
4217: %
4218: \expectation\left\{\frac{1}{n}\sum_{a\in{\cal C}}(\rdegree{}'_{a}-1)
4219: \right\} = \\
4220: \frac{\lnode'(1)}{\Gamma'(1)}\sum_{\rdegree}\Gamma_{\rdegree}\,
4221: \expectation_{\rdegree}\left\{\max(n_{\g}-1, 0)\,\ind_{n_\?=0}\right\}\, ,
4222: %
4223: \end{multline*}
4224: %
4225: where $\expectation_{\rdegree}$ denotes expectation with respect the
4226: multinomial variables $n_\0,n_\g,n_\?$ with sum $\rdegree$ and parameters
4227: $\lra_{\0}^{\infty},\lra_{\*}^{\infty},\lra_{\?}^{\infty}$.
4228: Once again, it is quite easy to compute the above expectations. One obtains
4229: %
4230: \begin{multline}
4231: %
4232: \expectation\left\{\frac{1}{n}\sum_{a\in{\cal C}}(\rdegree{}'_{a}-1)
4233: \right\} =
4234: \frac{\lnode'(1)}{\Gamma'(1)}\{
4235: \Gamma'(1-\lra_{\?})\lra_{\g}-\\
4236: -\Gamma(1-\lra_{\?})+\Gamma(1-\lra_{\?}-\lra_{\g})\}\, . \label{Cond2}
4237: %
4238: \end{multline}
4239: %
4240:
4241: By taking the difference of Eqs.~(\ref{Cond1}) and (\ref{Cond2}),
4242: and after a few algebraic manipulations, we finally get
4243: the desired result
4244: %
4245: \begin{eqnarray*}
4246: %
4247: \expectation [{\mathbb C}] = F(\lra,\cp,\gp)\, ,
4248: %
4249: \end{eqnarray*}
4250: %
4251: where
4252: %
4253: \begin{multline*}
4254: %
4255: F(\lra,\cp,\gp) \defas \lnode'(1)[\lra_\?(1-\rla_\?)-(\lra_\?+\lra_\g)
4256: (1-\rla_\?-\rla_\g)]-\\
4257: -\cp(1-\gp)[\lnode(\rla_\?+\rla_\g)-\lnode(\rla_\?)] + \\
4258: +\frac{\lnode'(1)}{\Gamma'(1)}[\Gamma(1-\lra_\?)-\Gamma(1-\lra_\?-\lra_\g)]\, .
4259: %
4260: \end{multline*}
4261: %
4262: Here we used the shorthand $\lra$ for the vector
4263: $(\lra_\?,\lra_\g,\lra_\0,\rla_\?,\rla_\g,\rla_\0)$.
4264:
4265: Imagine now changing $\gamma\to\gamma+\Delta\gamma$ and computing
4266: the number of new conditions on the newly guessed variables
4267: (whose expected number was computed in the previous section).
4268: Call $\Delta{\mathbb C}$
4269: the upper bound on their number provided by Lemma \ref{LemmaConditions}.
4270: It is clear that, repeating the above derivation, we get
4271: %
4272: \begin{multline*}
4273: %
4274: \expectation [\Delta {\mathbb C}] =
4275: F(\lra^{\infty}(\cp,\gp+\Delta\gp),\cp,\gp+\Delta\gp)-\\
4276: -F(\lra^{\infty}(\cp,\gp),\cp,\gp+\Delta\gp)\, ,
4277: %
4278: \end{multline*}
4279: %
4280: Consider now two separate possibilities. In the first case
4281: $\lra^{\infty}(\cp,\gp')$ is continuous (and therefore analytic) in the
4282: interval $\gp'\in [\gp,\gp+\Delta\gp]$. By Taylor expansion we get
4283: %
4284: \begin{eqnarray*}
4285: %
4286: \expectation [\Delta {\mathbb C}] = -
4287: \frac{\partial F}{\partial\lra}(\lra',\cp,\gp+\Delta\gp)
4288: \cdot
4289: \frac{\partial \lra^{\infty}(\cp,\gp)}{\partial\gp} \, \Delta\gamma
4290: + O((\Delta\gp)^2)\, .
4291: %
4292: \end{eqnarray*}
4293: %
4294: with the gradient of $F$ being evaluated at
4295: $\lra' = \lra^{\infty}(\cp,\gp+\Delta\gp)$. A direct calculation
4296: shows that the gradient vanishes at this point leading to
4297: $\expectation [\Delta {\mathbb C}] = O((\Delta\gp)^2)$.
4298:
4299: In the second case, the interval $[\gp,\gp+\Delta\gp]$ includes a
4300: discontinuity point (a jump) $\gp_{\rm j}$.
4301: Let $\lra_{{\rm j}+}\defas \lxl^{j+1} = \lim_{\gp\downarrow\gp_{\rm j}}
4302: \lra^{\infty}(\cp,\gp)$ and
4303: $\lra_{{\rm j}-} \defas \uxl^{j} = \lim_{\gp\uparrow\gp_{\rm j}}\lra^{\infty}(\cp,\gp)$. We have
4304: %
4305: \begin{eqnarray*}
4306: %
4307: \expectation [\Delta {\mathbb C}] =
4308: F(\lra_{{\rm j}+},\cp,\gp_{\rm j})- F(\lra_{{\rm j}-},\cp,\gp_{\rm j})+ O(\Delta\gp)\, .
4309: %
4310: \end{eqnarray*}
4311: %
4312:
4313: %Consider the $i^{\text{th}}$ variable node, and denote by ${X}_i(t)$
4314: %the number of $\g$ or $\0$ messages incoming at time $t$. (If there are no
4315: %$\g$ messages, confirmations correspond simply to redundant $\0$ messages.)
4316: %Observe that $0 \leq {X}_i(t)\leq \ldegree.$
4317: %
4318: %The number ${\mathbb{C}}_i(t)$ of new confirmations arising at node $i$ and time $t$
4319: %is
4320: %\begin{align*}
4321: %{\mathbb{C}}_i(t) =
4322: %\begin{cases}
4323: %X_i(t+1)-X_i(t), & \text{if~$X_i(t)>0$~or~$\chm{i}=\g$} \\
4324: %X_i(t+1)-X_i(t)-1, & \text{if~$X_i(t+1)>X_i(t)=0$}\\
4325: %& \;\;\;\;\text{and~$\chm{i}=\?$}.
4326: %\end{cases}
4327: %\end{align*}
4328: %%
4329: %In average, for a given variable of index $i$, this yields
4330: %\begin{multline*}
4331: %\expectation {\mathbb{C}}_i(t) = \lnode'(1)\left((\rla_\g^{t+1}+\rla_\0^{t+1} ) -(\rla_\g^t +\rla_\0^t )\right) \\
4332: % - \ih (1-p)\left(\lnode(\rla_\?^t) -\lnode(\rla_\?^{t+1})\right)\,,
4333: %
4334: %\end{multline*}
4335: %
4336: %where $\lnode'(1)=\frac{1}{\int \ledge}$ is the average left degree. The first term
4337: %is the average of $X_i(t+1)-X_i(t)$, while the second is $-1$ multiplied by the
4338: %probability of having $G_i(t+1)>G_i(t)=0$ and~$\chm{i}=\?$.
4339: %
4340: %In order to compute the {\em total} number of independent confirmations
4341: %appearing at iteration $t$, we must take into account potential
4342: %dependencies among confirmations appearing at distinct nodes.
4343: %A little thought shows ({\bf ?}) that such dependencies may only appear
4344: %among variable nodes which enter in the same parity-check. More precisely,
4345: %the unique possibility for such dependencies is that all the
4346: %messages from a given parity-check to the neighboring variables are $\g$
4347: %at time $t+1$, while this was not the case at time $t$. In this case, there is a unique independent
4348: %condition arising, while the sum of ${\mathbb{C}}_i(t)$ yields $\rdegree$.
4349: %To obtain the average number of new conditions per variable node
4350: %we must therefore subtract from $\expectation\, {\mathbb{C}}_i(t)$ the quantity
4351: %
4352: %\begin{align*}
4353: %\frac{\lnode'(1)}{\rnode'(1)}\sum_{\rdegree}\rnode_{\rdegree}(\rdegree-1)
4354: %[(\lra_\g^{t+1}+\lra_\0^{t+1})^{\rdegree}-(\lra_\g^{t}+\lra_\0^{t})^{\rdegree}]\, .
4355: %\end{align*}
4356: %
4357: %Therefore, we get the average number of independent confirmations per variable node, denoted by ${\mathbb{C}}$,
4358: %as
4359: %\begin{multline*}
4360: %{\mathbb{C}}(t) = \lnode'(1)\left((1-\rla_\?^{t+1}) -(1-\rla_\?^t)\right) \\
4361: % - \ih (1-p)\left(\lnode(\rla_\?^t) -\lnode(\rla_\?^{t+1})\right) \\
4362: %-\frac{\lnode'(1)}{\rnode'(1)}\sum_{\rdegree}\rnode_{\rdegree}(\rdegree-1)
4363: %[(1-\lra_\?^{t+1})^{\rdegree}-(1-\lra_\?^{t})^{\rdegree}]\,,
4364: %%
4365: %\end{multline*}
4366: %which, using $\redge(x)=\frac{\rnode'(x)}{\rnode'(1)}$, we can write as,
4367: %\begin{align*}
4368: %{\mathbb{C}}(t,\ih,\gp) & = F(\lra^{t+1},\ih,\gp) - F(\lra^t,\ih,\gp)\,,
4369: %
4370: %\end{align*}
4371: % where
4372: %$\lra^t$ is the triple $\lra^t\defas(\lra^t_\0,\lra^t_\g,\lra^t_\?)$ and
4373: %
4374: %\begin{multline*}
4375: %F(\lra,\ih,\gp)\defas
4376: %\lnode'(1)\left( \lra_\? (1-\rla_\?) \right)+ (1-\gp)\ih\lnode(\rla_\?)\\
4377: %+\frac{\lnode'(1)}{\rnode'(1)} \rnode(1-\lra_\?)\, .
4378: %\end{multline*}
4379: %
4380: %At the end of the process, the number of independent confirmations converges to
4381: %\begin{align*}
4382: %{\mathbb{C}}(\infty,\ih,\gp) & = F(\lra^{\infty},\ih,\gp) - F(\lra^0,\ih,\gp)\,.
4383: %
4384: %\end{align*}
4385: %The stationarity of $F$ with respect to variations of $\lra$
4386: %at DE fixed points comes directly from the fact that
4387: %$$
4388: %F'(\lra_\?) = \lnode'(1) \redge'(1-\lra_\?)\left(-\lra_\? + (1-\gp)\ih \ledge(1-\redge(1-\lra_\?)) \right)
4389: %$$
4390: %is equal to 0 at the fixed point. Therefore, using the same argument as for the
4391: %guess work ({\bf to finish; might be better to work directly with small deltas and not dp}), a change
4392: % d$\gp$ in guesses gives a $(\text{d}\gp)^2$ in contradictions as long as we do not operate
4393: %in the neighborhood of a jump.
4394: %
4395: %*******************************************************************
4396: %
4397: \subsection{Finishing the proof}
4398:
4399: Consider now the guessing strategy explained in Section \ref{sec:guessstrat}. First
4400: the received message is decoded with the usual iterative decoder.
4401: At this point $\gamma = 0$.
4402: Then each bit is selected independently
4403: with $\Delta\gamma/(1-\gamma)$ and guessed if its valued was not determined
4404: (eventually in terms of former guesses) at previous stages.
4405: The \maxwell decoder is then run until a fixed point is reached.
4406: The number of new guesses at this stage is $\Delta{\mathbb G}_{\gamma}$ and
4407: the number of new conditions is upper bounded by $\Delta{\mathbb C}_{\gamma}$.
4408: This operation is repeated until $\nu_i(\infty) \in\{\0, \g\}$ for each $i$.
4409: Without loss of generality, we may imagine this to happen at
4410: $\gp = 1$.
4411:
4412: At this point each realization of the guesses compatible with the
4413: conditions yields a codeword compatible with the received message.
4414: We have
4415: %
4416: \begin{multline*}
4417: %
4418: \lim_{n\to\infty}\frac{1}{n}\expectation_{\graph}[ H_{\graph}(X|Y)] \ge
4419: \sum_{\gamma}\expectation[\Delta{\mathbb G}_{\gamma}] -
4420: \sum_{\gamma}\expectation[\Delta{\mathbb C}_{\gamma}] \\
4421: =\int_{0}^{1} \cp\lnode(\rla_\?(\gp,\cp))\, \de\gp
4422: -\sum_{\gamma_{\rm j}}\Delta F_{\rm j}
4423: + O(\Delta\gp)\, ,
4424: %
4425: \end{multline*}
4426: %
4427: where the last sums runs over the jump positions $\gamma_{{\rm j}}$ and
4428: $\Delta F_{\rm j}\le
4429: F(\lra_{{\rm j}+},\cp,\gp_{\rm j})- F(\lra_{{\rm j}-},\cp,\gp_{\rm j})$
4430: is the discontinuity of $F$ at those positions.
4431: In order to finish the proof of Lemma \ref{ImprovedUB}, notice that
4432: $H(X|Y)$ does not depend upon $\Delta\gp$ and we can therefore take
4433: the limit $\Delta\gp\to 0$ discarding $O(\Delta\gp)$ terms.
4434: Moreover $\rla_{\?}(\gp,\cp) = \rla(\cp(1-\gp))$ (the last
4435: quantity being the fixed point of DE for the usual BP decoder at
4436: erasure probability $\cp$), and therefore
4437: %
4438: \begin{align*}
4439: %
4440: \int_{0}^{1} \cp\lnode(\rla_\?(\gp,\cp)) \, \de\gp =
4441: \int_{0}^{\cp} \lnode(\rla(\cp')) \, \de\cp'\,
4442: %
4443: \end{align*}
4444: %
4445: is just the area under the BP \exitentropy curve
4446: (dark gray in Fig.~\ref{fig:exitcurve}, (a)). Finally,
4447: let $\cp_{\rm j} = (1-\gp_{\rm j})\cp$ and
4448: $(\lra(\cp_{\rm j}+),\rla(\cp_{\rm j}+))$
4449: and $(\lra(\cp_{\rm j}-),\rla(\cp_{\rm j}-))$ be the fixed point of
4450: DE for the usual iterative decoder just above and below the jump.
4451: Then
4452: %
4453: \begin{eqnarray*}
4454: %
4455: \Delta F_{\rm j} = P_{\cp_{\rm j}}(\lra(\cp_{\rm j}-),\rla(\cp_{\rm j}-)
4456: -P_{\cp_{\rm j}}(\lra(\cp_{\rm j}+),\rla(\cp_{\rm j}+))\, ,
4457: %
4458: \end{eqnarray*}
4459: %
4460: where $P_\cp(\lra,\rla)$ is the trial entropy, cf. Def.~\ref{TrialDef}.
4461: Because of Lemma~\ref{TrialLemma}, $\Delta F_{\rm j}$ is just the area
4462: delimited by the EBP \exitentropy curve and a vertical line through the jump,
4463: (dark gray in Fig.~\ref{fig:exitcurve}, (b)).
4464: %
4465: %*******************************************************************
4466: %
4467: \subsection{Maxwell Decoder: Illustration and Implementation}
4468: The Maxwell decoder provides an {\em interpretation} for the balance of areas
4469: which we described in Sections \ref{UpperSection} and \ref{sec:Counting}.
4470: For many ensembles, e.g., the $(3,6)$-regular ensemble,
4471: Theorem \ref{CountingTheorem} gives a complete characterization
4472: of the MAP EXIT function and therefore a complete justification
4473: of the Maxwell construction. In some other cases we are not quite
4474: as lucky, see e.g. the ensemble discussed in Example \ref{equ:doublejumpexample},
4475: and we can only conjecture that the parts
4476: of the MAP EXIT function which are not covered by Theorem \ref{CountingTheorem}
4477: also follow the Maxwell construction. Let us now review some
4478: typical case.
4479:
4480: %Example \ref{ex:prof36}. We can also complete this picture by conjecturing that
4481: %the Maxwell construction applies locally at each phase transition. Simulations
4482: %validate this conjecture at the second phase transition for our running
4483: %double-jump example. See Example \ref{ex:prof2j}.
4484: \bex[$(3,6)$ LDPC ensemble] \label{ex:prof36} Consider the
4485: \ddp $(\ledge,\redge)=(x^2,x^5)$ and the
4486: corresponding $\ldpc$ ensemble with design rate one-half.
4487: Its BP and MAP \exitentropy functions are depicted in
4488: Fig.~\ref{fig:exitcurve} together with
4489: the balance conditions.
4490: Fig.~\ref{fig:guesscont} shows the evolution of the
4491: entropy $\entropy(\iter)$, i.e., the logarithm of the number of running copies
4492: as discussed in Fig. \ref{fig:maxwelltwodecoder},
4493: as a function of the fraction of bits determined by the decoding
4494: process for the $(3, 6)$-regular LDPC ensemble.
4495: %(This fraction of bits equals $1-\ih(\xl_\iter)\ledge(\xr_\iter)$
4496: %and is increasing with respect to $\iter$).
4497: Transmission takes
4498: place over BEC($\ih = 0.46$), i.e., we fix the channel parameter $\ih$ so that
4499: $\ih^\BP\approx0.4294<\ih<\ih^\MAP\approx0.4882$. After transmission,
4500: a fraction $1-\ih=0.54$ of bits is known. The classical BP algorithm
4501: proceeds until it gets stuck at the fixed point
4502: $(\xl^\ih \approx 0.3789, \xr^\ih \approx 0.9076)$
4503: of DE. At this point (point A in the figure),
4504: a fraction $1-\ih\lnode(\xr^\ih)\approx0.6561$ of bits has been determined.
4505: Now the guessing phase of the \maxwell decoder starts.
4506: It ends at point B, which corresponds to the BP threshold
4507: $(\xl^\BP \approx 0.2606, \xr^\BP \approx 0.7790)$.
4508: The total fraction of guesses that the \maxwell decoder has to venture is
4509: $\int_{\xl^\BP}^{\xl^\ih}\xh(\ih(\xl))\text{d}\ih(\xl)=P(\xl^\ih,\xr^\ih)-
4510: P(\xl^\BP,\xr^\BP)$.
4511: For our specific example we have
4512: $P(\xl,\xr(\xl))=-\frac{5\xl^2}{2}+10\xl^3-\frac{25\xl^4}{2}+7\xl^5-\frac{3\xl^6}{2}$,
4513: so that the total fraction of guesses is equal to $0.0201509$.
4514: For a blocklength of $\n=34000$ this corresponds to roughly $685$ guesses.
4515: At this point the BP decoding phase resumes. More and more
4516: guesses are confirmed. Since we are operating below the MAP threshold,
4517: (essentially) all guesses are eventually confirmed and the
4518: \maxwell decoder comes to a halt.
4519: \begin{figure}[htp]
4520: \begin{center}
4521: \begin{picture}(240,130)
4522: \put(50,5)
4523: {
4524: \put(-67,-10){\includegraphics[scale=0.62]{36instLaTeX}}
4525: \put(10,15){\includegraphics[scale=0.3]{prof_explanation_Abis}}
4526: \put(30,80){\includegraphics[scale=0.3]{prof_explanation_Bbis}}
4527: \put(135,70){\includegraphics[scale=0.3]{prof_explanation_Cbis}}
4528: \put(149,11){\includegraphics[scale=0.3]{prof_explanation_Dbis}}
4529: \put(68,40){\makebox(0,0){$\rightarrow$}}
4530: \put(91,94){\makebox(0,0){$\searrow$}}
4531: \put(138,64){\makebox(0,0){$\swarrow$}}
4532: \put(144,8){\makebox(0,0){$\swarrow$}}
4533: %\put(148,2){\makebox(0,0){$\longrightarrow$}}
4534: %\put(160,-0){\makebox(0,0){${{\text{\small residual~graph}}}$}}
4535: %\put(-50,111){\rotatebox{90}{\makebox(0,0){${{\text{\small entropy}}}$}}}
4536: \put(-55,101){\makebox(0,0){$\entropy$}}
4537: \put(70,8){\makebox(0,0){A}}
4538: \put(103,82){\makebox(0,0){B}}
4539: }
4540: \put(0,0)
4541: {
4542: \put(2,79){\makebox(0,0)[l]{{$680$}}}
4543: \put(2,61){\makebox(0,0)[l]{{$510$}}}
4544: \put(2,43){\makebox(0,0)[l]{{$340$}}}
4545: \put(2,25){\makebox(0,0)[l]{{$170$}}}
4546: \put(-5,1){\makebox(0,0)[l]{{$0.0$}}}
4547: \put(30,1){\makebox(0,0)[l]{{$0.2$}}}
4548: \put(68,1){\makebox(0,0)[l]{{$0.4$}}}
4549: \put(110,1){\makebox(0,0)[l]{{$0.6$}}}
4550: \put(148,1){\makebox(0,0)[l]{{$0.8$}}}
4551: \put(190,1){\makebox(0,0)[l]{{$1.0$}}}
4552: }
4553: \end{picture}
4554: \caption{\maxwell decoder applied to the $(3,6)$-regular LDPC ensemble.
4555: Asymptotic entropy of the \maxwell decoder $\entropy$
4556: (logarithm of the number of running copies) as a function of the
4557: fraction of determined bits. 15
4558: channel and code realizations with $\ih=0.46$ and blocklength
4559: $\n=34\cdot 10^3$ are shown (dashed curves) together with the analytic
4560: asymptotic curve (solid curve). The inserts show how the entropy
4561: curve can be constructed from the \exitentropy curve. The fraction of guesses
4562: is shown in the 2 left-most inserts while the fraction of
4563: contradictions is shown in the 2 right inserts. }
4564: \label{fig:guesscont}
4565: \end{center}
4566: \end{figure}
4567:
4568:
4569: \begin{figure}[htp]
4570: \begin{center}
4571: \begin{picture}(240,100)
4572: \put(50,0)
4573: {
4574: \put(10,0){\includegraphics[scale=0.5]{36convLaTeX}}
4575: \put(10,101){\makebox(0,0)[l]{$\frac{\entropy}{\n\cdot10^{-3}}$ }}
4576: \put(10,78){\makebox(0,0)[l]{\footnotesize{$25$}}}
4577: \put(10,64){\makebox(0,0)[l]{\footnotesize{$20$}}}
4578: \put(10,51){\makebox(0,0)[l]{\footnotesize{$15$}}}
4579: \put(10,37){\makebox(0,0)[l]{\footnotesize{$10$}}}
4580: \put(14,23){\makebox(0,0)[l]{\footnotesize{$5$}}}
4581: \put(14,5){\makebox(0,0)[l]{\footnotesize{$0.4$}}}
4582: \put(46,5){\makebox(0,0)[l]{\footnotesize{$0.6$}}}
4583: \put(78,5){\makebox(0,0)[l]{\footnotesize{$0.8$}}}
4584: \put(110,5){\makebox(0,0)[l]{\footnotesize{$1.0$}}}
4585: %\put(68,-3){\makebox(0,0){{\small{(b)}}}}
4586: %
4587: }
4588: \end{picture}
4589: \caption{\maxwell decoder applied to the $(3,6)$ LDPC ensemble:
4590: Expected symptotic entropy as a function of the fraction of determined bits at $\ih=0.46$ (solid curve)
4591: and empirical average entropy curves (gray curves). Simulations are shown for $\n=780$ (average over $6\cdot10^4$ realizations), $\n=3125$ (average over $16\cdot10^3$ realizations), $\n=12500$ (average over $4\cdot10^3$ realizations),
4592: $\n=50000$ (average over $10^3$ realizations), $\n=200000$ (average over $150$ realizations).
4593: }
4594: \label{fig:guesscontbis}
4595: \end{center}
4596: \end{figure}
4597: \eex
4598:
4599: \bex[Typical Double ``Jump''] \label{ex:prof2j}
4600: %\bex[Typical Double ``Jump'']
4601: \label{ex:running2jumps}
4602: Consider the
4603: \ddp $(\ledge,\redge)=(\frac{3x + 3 x^2+4x^{13}}{10},x^6)$ and the
4604: corresponding $\ldpc$ ensemble with design rate $\drate=\frac{19}{39}\approx0.4872$.
4605: %
4606: Its BP \exitentropy function is depicted in
4607: Fig.~\ref{fig:epsilon}, its EBP \exitentropy curve together with
4608: the balance conditions is shown in
4609: Fig.~\ref{fig:multijump}. Finally, in Example \ref{equ:doublejumpexample} we have
4610: discussed how large parts of the MAP \exitentropy curve can be constructed
4611: based on Theorem \ref{CountingTheorem}.
4612: %Let us now explain how the
4613: %Maxwell decoder can be used to fill in the remainder of the $\xh^\MAP$.
4614: %
4615: %Assume we use the \maxwell decoder with $\ih=1$. Then we know from
4616: %the analysis of the \maxwell decoder that
4617: %the entropy at channel parameter $\ih$ is upper bounded by
4618: %\begin{align*}
4619: %H(X \mid Y(\ih)) \leq \min_{(\xl, \xr)} P(\xl,\xr),
4620: %\end{align*}
4621: %where the minimum is taken over all fixed-point pairs $(\xl, \xr)$ of density
4622: %evolution such that $\ih=\ih(\xl)$. If, for some $\ih$, we have equality
4623: %then we must have equality for all channel parameters above $\ih$.
4624: %In such a situation we can then take the derivative to construct
4625: %the MAP EXIT curve $\xh^\MAP$. In particular, if Theorem \ref{CountingTheorem}
4626: %applies at $\ih^\MAP$, then we know from the \maxwell decoder the whole
4627: %MAP EXIT curve.
4628: %What we conjectured to be the the MAP \exitentropy curve is depicted (zoomed) in
4629: %Fig.~%\ref{fig:exitconclusion}. \ref{fig:multijump}
4630: The MAP threshold is $\ih^\MAP\approx0.4913$
4631: (at $\xl^\MAP\approx0.1434$). According to the Maxwell costruction,
4632: the second MAP discontinuities occurs at
4633: $\ih^{\MAP,2}\approx0.5186$ (at $\lxl^{\MAP,2}\approx0.2378$, $\uxl^{\MAP,2}\approx0.4121$) .
4634:
4635:
4636: Fig.~\ref{fig:guesscont2J} shows the evolution of the
4637: entropy $\entropy(\iter)$ for $\ih=0.5313$. This corresponds
4638: to the point $C$ in Fig.~\ref{equ:doublejumpexample}, the
4639: first point at which the counting argument no longer applies.
4640: %Observe that the entropy profile exhibits two distinct phases,
4641: %each one corresponding to a particular ``jump''. Although the counting argument
4642: %does no longer apply at point C of Fig. \ref{fig:2Jebpexit}, we conjecture that the Maxwell construction
4643: %``applies'' at each ``jump'', in particular at the second phase transition. This
4644: %conjecture, when extended to general multiple-jump ensembles, complete the
4645: %picture of multiple and local Maxwell construction which is the main
4646: %picture we convey all along this paper.
4647: By comparing the result of the simulations to the analytic curve,
4648: corresponding to the Maxwell construction we can see that
4649: at least emperically the Maxwell construction seems to be
4650: valid over the whole range.
4651: %For our running example, simulations
4652: %validate this conjecture: indeed it can be observed in (b) that the empirical
4653: %averaged curve converge to this analytic curve.
4654: %Moreover, this convergence appears empirically to be of order $1/\sqrt{\n}$.
4655: % It can be observed in (a)
4656: %that the
4657: %concentration of particular instances is weak around the second phase transition.
4658: \begin{figure}[htp]
4659: \begin{center}
4660: \begin{picture}(240,100)
4661: \put(-5,0){
4662: \put(10,0){\includegraphics[scale=0.5]{2JinstLaTeX}}
4663: \put(14,101){\makebox(0,0)[l]{\footnotesize$\entropy$ }}
4664: \put(6,78){\makebox(0,0)[l]{\footnotesize{$850$}}}
4665: \put(6,64){\makebox(0,0)[l]{\footnotesize{$680$}}}
4666: \put(6,51){\makebox(0,0)[l]{\footnotesize{$510$}}}
4667: \put(6,37){\makebox(0,0)[l]{\footnotesize{$340$}}}
4668: \put(6,23){\makebox(0,0)[l]{\footnotesize{$170$}}}
4669: \put(14,5){\makebox(0,0)[l]{\footnotesize{$0.4$}}}
4670: \put(46,5){\makebox(0,0)[l]{\footnotesize{$0.6$}}}
4671: \put(78,5){\makebox(0,0)[l]{\footnotesize{$0.8$}}}
4672: \put(110,5){\makebox(0,0)[l]{\footnotesize{$1.0$}}}
4673: \put(68,-3){\makebox(0,0){{\small{(a)}}}}
4674:
4675: }
4676:
4677: \put(120,0)
4678: {
4679: \put(10,0){\includegraphics[scale=0.5]{2JconvLaTeX}}
4680: \put(10,101){\makebox(0,0)[l]{$\frac{\entropy}{\n\cdot10^{-3}}$ }}
4681: \put(10,78){\makebox(0,0)[l]{\footnotesize{$25$}}}
4682: \put(10,64){\makebox(0,0)[l]{\footnotesize{$20$}}}
4683: \put(10,51){\makebox(0,0)[l]{\footnotesize{$15$}}}
4684: \put(10,37){\makebox(0,0)[l]{\footnotesize{$10$}}}
4685: \put(14,23){\makebox(0,0)[l]{\footnotesize{$5$}}}
4686: \put(14,5){\makebox(0,0)[l]{\footnotesize{$0.4$}}}
4687: \put(46,5){\makebox(0,0)[l]{\footnotesize{$0.6$}}}
4688: \put(78,5){\makebox(0,0)[l]{\footnotesize{$0.8$}}}
4689: \put(110,5){\makebox(0,0)[l]{\footnotesize{$1.0$}}}
4690: \put(68,-3){\makebox(0,0){{\small{(b)}}}}
4691:
4692: }
4693: \end{picture}
4694: \caption{\maxwell decoder applied to the irregular ``double-jump'' LDPC ensemble shown in Fig.~\ref{fig:epsilon}:
4695: Asymptotic entropy as a function of the fraction of determined bits at $\ih=0.5313$ (point $B$). (a) 15 channel
4696: and code realizations of blocklength $\n=34000$ are shown (dashed curves) together with the analytic
4697: asymptotic curve (solid curve). (b) Convergence of the average entropy curves (gray curves) to the analytic
4698: expected curve (solid curve). Simulations are shown for $\n=780$ (average over $6\cdot10^4$ realizations), $\n=3120$ (average over $16\cdot10^3$ realizations), $\n=12480$ (average over $4\cdot10^3$ realizations),
4699: $\n=50017$ (average over $10^3$ realizations), $\n=200500$ (average over $250$ realizations).
4700: }
4701: \label{fig:guesscont2J}
4702: \end{center}
4703: \end{figure}
4704:
4705:
4706: \eex
4707:
4708: \section{Some Further Examples}
4709: \subsection{Special Cases}
4710: %The \exitentropy functions which are presented in the sequel illustrate
4711: %some typical cases. First, let us deal with the
4712: %example introduced in Fig.~\ref{fig:epsilon}.
4713: Although (for sake of simplicity) we did not
4714: discuss this case in the previous sections,
4715: other curious (but frequent) examples are those
4716: when the number of discontinuities $J^\BP$
4717: of the BP \exitentropy curves is not equal to the number of
4718: discontinuities $J^\MAP$ of the MAP \exitentropy curve.
4719: Examples \ref{ex:mapleqbp} and \ref{ex:bpleqmap} show two such cases.
4720:
4721: \bex[$J^\MAP<J^\BP$] \label{ex:mapleqbp} Consider the
4722: \ddp $(\ledge,\redge)=(\frac{x^{10}+x^{60}}{2},\frac{3 x^{10}+17 x^{80}}{20})$
4723: and the corresponding $\ldpc$ ensemble with rate $\drate=\frac{3209}{5832}\approx0.5502$.
4724:
4725: \begin{figure}[htp]
4726: \centering
4727: \setlength{\unitlength}{1bp}
4728: \begin{picture}(240,140)
4729: \put(5,28)
4730: {
4731: \put(5,5){\includegraphics[width=100bp]{maliciousMAPleqBP.ps}}
4732: \put(135,5){\includegraphics[width=100bp]{maliciousMAPleqBPconst.ps}}
4733: % coordinates
4734: \put(0,0){\makebox(0,0){\small{$0$}}}
4735: \put(100,0){\makebox(0,0){\small{$1$}}}
4736: \put(0,100){\makebox(0,0){\small{$1$}}}
4737: \put(130,0){\makebox(0,0){\small{$0$}}}
4738: \put(230,0){\makebox(0,0){\small{$1$}}}
4739: \put(130,100){\makebox(0,0){\small{$1$}}}
4740: % EXIT in (a)
4741: \put(40,-5){\makebox(0,0){\small{$\ih^\BP$}}}
4742: % EXIT in (b)
4743: \put(179,-5){\makebox(0,0){\small{$\ih^\MAP$}}}
4744: \put(200,80){\makebox(0,0){\small{$~$}}}
4745: }
4746: % caption
4747: \put(60,7){\makebox(0,0){{\small{(a)}}}}
4748: \put(190,7){\makebox(0,0){{\small{(b)}}}}
4749: \end{picture}
4750: \caption{
4751: When the numbers of BP and MAP ``jumps'' (respectively,
4752: $J^\BP$ and $J^\MAP$) are different:
4753: (a) BP \exitentropy function with $J^\BP=2$ (b) MAP \exitentropy function
4754: with $J^\MAP=1$ and Maxwell constriction.
4755: }
4756: \label{fig:malicious2}
4757: \end{figure}
4758: The MAP \exitentropy curve has a single ``jump'' at
4759: $\ih^\MAP\approx0.4493$ ($\xl^\MAP\approx0.4425$)
4760: whereas the BP \exitentropy curve has two such
4761: singularities at $\ih^\BP\approx0.2941$ ($\xl^\BP\approx0.05738$)
4762: and $\ih^{(\BP,2)}\approx0.3254$ ($\uxl^{(\BP,2)}\approx0.2117$) as
4763: shown in Fig.~\ref{fig:malicious2}.
4764: %\begin{figure}[htp]
4765: %\begin{center}
4766: %\begin{picture}(240,100)
4767: %
4768: %\put(10,0)
4769: %{
4770: %\put(10,0){\includegraphics[scale=0.5]{2bp1mp-malicious}}
4771: %%\put(10,101){\makebox(0,0)[l]{$\frac{\entropy}{\n\cdot10^{-3}}$ }}
4772: %\put(10,78){\makebox(0,0)[l]{\footnotesize{$25$}}}
4773: %\put(10,64){\makebox(0,0)[l]{\footnotesize{$20$}}}
4774: %\put(10,51){\makebox(0,0)[l]{\footnotesize{$15$}}}
4775: %\put(10,37){\makebox(0,0)[l]{\footnotesize{$10$}}}
4776: %\put(14,23){\makebox(0,0)[l]{\footnotesize{$5$}}}
4777: %\put(14,5){\makebox(0,0)[l]{\footnotesize{$0.4$}}}
4778: %\put(46,5){\makebox(0,0)[l]{\footnotesize{$0.6$}}}
4779: %\put(78,5){\makebox(0,0)[l]{\footnotesize{$0.8$}}}
4780: %\put(110,5){\makebox(0,0)[l]{\footnotesize{$1.0$}}}
4781: %\put(68,-3){\makebox(0,0){{\small{(b)}}}}
4782: %
4783: %}
4784: %\end{picture}
4785: %\caption{\maxwell decoder applied to our running ``$J^\MAP<J^\BP$'' LDPC ensemble at $\ih=0.4$. The
4786: %average over 30 channel and code realizations of blocklength $\n=30020$ is shown (gray curve) together with the analytic
4787: %asymptotic curve (solid curve).{\bf{PostScript not done!}}
4788: %}
4789: %\label{fig:2bp1map-conj}
4790: %\end{center}
4791: %\end{figure}
4792: %\begin{floatingfigure}{80bp}
4793: \begin{figure}[htp]
4794: \centering
4795: \setlength{\unitlength}{1bp}
4796: \begin{picture}(80,70)
4797: \put(10,0){\includegraphics[width=60bp]{WeightDistribution1bp2map04493}}
4798: \put(10,0)
4799: {
4800: \put(58,-3){\makebox(0,0){$\scriptstyle{\frac32}$}}
4801: \put(-3,58){\makebox(0,0){$\scriptstyle{0}$}}
4802: \put(19,-3){\makebox(0,0){$\scriptstyle{\frac12}$}}
4803: %\put(39,-3){\makebox(0,0){$\scriptstyle{1}$}}
4804: \put(-6,29){\makebox(0,0){$\scriptstyle{-\frac{1}{40}}$}}
4805: \put(-6,0){\makebox(0,0){$\scriptstyle{-\frac{1}{20}}$}}
4806: }
4807: \end{picture}
4808: \caption{
4809: Function $\Psi_{\nddp}(u)$ for the \ddp formed by the residual
4810: ensemble at $\ih^\MAP=0.4493$.
4811: }
4812: \label{fig:wd1bp2map}
4813: \end{figure}
4814: %\end{floatingfigure}
4815: As shown in Fig.~\ref{fig:wd1bp2map},
4816: Theorem \ref{CountingTheorem} applies at the MAP
4817: threshold and so the whole MAP EXIT curve is determined by the counting
4818: argument in this case. The Maxwell construction is therefore confirmed
4819: in this case.
4820: \eex
4821:
4822: \bex[$J^\BP<J^\MAP$] \label{ex:bpleqmap}
4823: Consider the \ddp $(\ledge,\redge)=(\frac{3x+3x^2+14x^{50}}{20},x^{15})$
4824: and the corresponding $\ldpc$ ensemble with design rate $\drate=\frac{311}{566}\approx0.5495$.
4825: \begin{figure}[htp]
4826: \centering
4827: \setlength{\unitlength}{1bp}
4828: \begin{picture}(240,140)
4829: \put(5,28)
4830: {
4831: \put(5,5){\includegraphics[width=100bp]{maliciousBPleqMAP.ps}}
4832: \put(135,5){\includegraphics[width=100bp]{maliciousBPleqMAPconst.ps}}
4833: % coordinates
4834: \put(0,0){\makebox(0,0){\small{$0$}}}
4835: \put(100,0){\makebox(0,0){\small{$1$}}}
4836: \put(0,100){\makebox(0,0){\small{$1$}}}
4837: \put(130,0){\makebox(0,0){\small{$0$}}}
4838: \put(230,0){\makebox(0,0){\small{$1$}}}
4839: \put(130,100){\makebox(0,0){\small{$1$}}}
4840: % EXIT in (a)
4841: \put(40,-5){\makebox(0,0){\small{$\ih^\BP$}}}
4842: \put(54,105){\makebox(0,0){\small{A}}}
4843: % EXIT in (b)
4844: \put(179,-5){\makebox(0,0){\small{$\ih^\MAP$}}}
4845: \put(200,80){\makebox(0,0){\small{$~$}}}
4846: \put(185,105){\makebox(0,0){\small{A}}}
4847: }
4848: % caption
4849: \put(60,7){\makebox(0,0){{\small{(a)}}}}
4850: \put(190,7){\makebox(0,0){{\small{(b)}}}}
4851: \end{picture}
4852: \caption{
4853: When the numbers of BP and MAP ``jumps'' (respectively, $J^\BP$ and $J^\MAP$) are different:
4854: (a) BP \exitentropy function with $J^\BP=1$ (b) MAP
4855: \exitentropy function with $J^\MAP=2$ and Maxwell construction.
4856: }
4857: \label{fig:malicious1}
4858: \end{figure}
4859: The BP \exitentropy curve has a single ``jump'' at
4860: $\ih^\BP\approx0.3531$ ($\xl^\BP\approx0.3008$).
4861:
4862: Unfortunately, Theorem \ref{CountingTheorem} shows
4863: the tightness of the \maxwell construction
4864: only up to point A (at $\ih\approx0.5063$, see Fig.~\ref{fig:malicious1}) .
4865: But it is
4866: quite natural to conjecture that the MAP \exitentropy curve
4867: has two singularities, namely
4868: at $\ih^\MAP\approx0.3986$ ($\xl^\MAP\approx0.0340$)
4869: and at $\ih^{(\MAP,2)}\approx0.4855$ ($\uxl^{(\MAP,2)}\approx0.1096$)
4870: as shown in Fig.~\ref{fig:malicious1}. This is validated by the \maxwell
4871: decoder. Namely the \maxwell decoder
4872: gives a residual entropy (as a fraction of the blocklength) of
4873: $\frac{\entropy}{\n}\approx0.0121$ at $\ih=0.44$. This value is
4874: exactly the value of the area (between $\ih=0$ and $\ih=0.44$) under
4875: the conjectured MAP EXIT curve. This shows that, between the two
4876: conjectured MAP phase transitions, the \maxwell decoder follows
4877: the part of the EBP EXIT function which is ``hidden'' from the
4878: BP decoder.
4879: %\begin{figure}[htp]
4880: %\begin{center}
4881: %\begin{picture}(240,100)
4882: %
4883: %\put(10,0)
4884: %{
4885: %\put(10,0){\includegraphics[scale=0.5]{1bp2mp-malicious}}
4886: %%\put(10,101){\makebox(0,0)[l]{$\frac{\entropy}{\n\cdot10^{-3}}$ }}
4887: %\put(10,78){\makebox(0,0)[l]{\footnotesize{$25$}}}
4888: %\put(10,64){\makebox(0,0)[l]{\footnotesize{$20$}}}
4889: %\put(10,51){\makebox(0,0)[l]{\footnotesize{$15$}}}
4890: %\put(10,37){\makebox(0,0)[l]{\footnotesize{$10$}}}
4891: %\put(14,23){\makebox(0,0)[l]{\footnotesize{$5$}}}
4892: %\put(14,5){\makebox(0,0)[l]{\footnotesize{$0.4$}}}
4893: %\put(46,5){\makebox(0,0)[l]{\footnotesize{$0.6$}}}
4894: %\put(78,5){\makebox(0,0)[l]{\footnotesize{$0.8$}}}
4895: %\put(110,5){\makebox(0,0)[l]{\footnotesize{$1.0$}}}
4896: %\put(68,-3){\makebox(0,0){{\small{(b)}}}}
4897: %
4898: %}
4899: %\end{picture}
4900: %\caption{\maxwell decoder applied to our running ``$J^\BP<J^\MAP$'' LDPC ensemble at $\ih=0.44$. The
4901: %average over 1 channel and code realizations of blocklength $\n=30020$ is shown (gray curve) together with the analytic
4902: %asymptotic curve (solid curve).{\bf{PostScript not done!}}
4903: %}
4904: %\label{fig:2bp1map-conj}
4905: %\end{center}
4906: %\end{figure}
4907: %
4908: %Again our conjecture is natural and simulations shown in Fig. \ref{fig:1bp2map-conj}
4909: %seem to confirm its correctness.
4910: The Maxwell construction is conjectured to hold in this case.
4911: \eex
4912:
4913:
4914:
4915:
4916:
4917:
4918:
4919:
4920:
4921: \subsection{Difference Between MAP and BP Threshold}
4922: Let $\drate<1$ be the design rate. Consider a sequence of degree distribution pairs
4923: $\{(\ledge(x),\redge(x))=(x^{\ldegree-1},x^{\frac{\ldegree}{1-\drate}-1})\}_{\ldegree\geq2}$
4924: with fixed design rate $\drate$. Ensembles associated to this sequence are regular LDPC
4925: code ensembles. We have seen in
4926: Fact~\ref{fact:regular} that such ensembles have at most one jump and
4927: therefore we expect our bound on the MAP threshold to be tight.
4928: It was shown already in \cite{Mac97}, that if $\ldegree$ is increased then
4929: the weight distribution of such ensembles converges to the one
4930: of Shannon's random ensemble and, hence, the MAP threshold of
4931: such ensembles converges to the Shannon limit.
4932: Using the replica method,
4933: an explicit asymptotic expansion of the MAP threshold was given
4934: in \cite{Andrea}.
4935:
4936: Let us give here an alternative proof of this fact using our
4937: machinery.
4938: That the MAP threshold $\ih^\MAP(\ldegree)$ converges to the Shannon threshold is shown
4939: in Fact~\ref{theo:randomregularml}.
4940: On the other hand, as stated
4941: in Fact~ \ref{theo:randomregular}, the
4942: BP threshold $\ih^\MAP(\ldegree)$ goes to 0 when $\ldegree\to\infty$.
4943: This shows that the two thresholds can be arbitrarily far apart, and nevertheless
4944: the MAP \exitentropy curve can be constructed from the corresponding (E)BP \exitentropy curve!
4945:
4946: This is illustrated in Fig.~\ref{fig:regcodes} and the proofs are given in the sequel.
4947: \begin{figure}[htp]
4948: \centering
4949: \setlength{\unitlength}{1bp}%
4950: \begin{picture}(240,140)
4951: \put(5,20)
4952: {
4953: \put(5,5){\includegraphics[width=100bp]{EPSregcodes}}
4954: \put(135,5){\includegraphics[width=100bp]{BPEXITregcodes}}
4955: % coordinates
4956: \put(0,0){\makebox(0,0){\small{$0$}}}
4957: \put(100,0){\makebox(0,0){\small{$1$}}}
4958: \put(0,100){\makebox(0,0){\small{$1$}}}
4959: \put(130,0){\makebox(0,0){\small{$0$}}}
4960: \put(230,0){\makebox(0,0){\small{$1$}}}
4961: \put(130,100){\makebox(0,0){\small{$1$}}}
4962: % epsilon
4963: \put(-7,42){\makebox(0,0){\scriptsize{$\ih^{\BP}(2)$}}}
4964: \put(-9,33){\makebox(0,0){\scriptsize{$=\ih^{\text{SC}}$}}}
4965: \put(33,48){\makebox(0,0){\scriptsize{$\ih^\BP(3)$}}}
4966: %\put(28,38){\makebox(0,0){\scriptsize{$\ih^\BP$}}}
4967: %\put(23,28){\makebox(0,0){\scriptsize{$\ih^\BP(6)$}}}
4968: \put(31,28){\makebox(0,0){\scriptsize{$\ih^\BP(12)$}}}
4969: \put(31,17){\makebox(0,0){\scriptsize{$\ih^\BP(35)$}}}
4970: \put(24,9){\makebox(0,0){\scriptsize{$\ih^\BP(70)$}}}
4971: \put(60,80){\makebox(0,0){\small{$\ih(\xl)$}}}
4972: % EXIT
4973: \put(170,-2){\makebox(0,0){\scriptsize{$\ih^\BP(2)$}}}
4974: \put(185,45){\makebox(0,0){\scriptsize{$\ih^\BP(3)$}}}
4975: \put(178,54){\makebox(0,0){\scriptsize{$\ih^\BP(4)$}}}
4976: \put(170,65){\makebox(0,0){\scriptsize{$\ih^\BP(6)$}}}
4977: \put(162,75){\makebox(0,0){\scriptsize{$\ih^\BP(12)$}}}
4978: \put(153,85){\makebox(0,0){\scriptsize{$\ih^\BP(35)$}}}
4979: \put(148,94){\makebox(0,0){\scriptsize{$\ih^\BP(70)$}}}
4980: \put(185,105){\makebox(0,0){\small{$\ih^\MAP$}}}
4981: \put(210,80){\makebox(0,0){\small{$\xh(\ih)$}}}
4982: }
4983: % caption
4984: \put(60,7){\makebox(0,0){{\small{(a)}}}}
4985: \put(190,7){\makebox(0,0){{\small{(b)}}}}
4986: \end{picture}
4987: \caption{Regular BP \exitentropy entropy curves with design rate $\drate=\frac{1}{2}$. (a) Channel entropy function $\xl\mapsto\ih^{(\ldegree)}(\xl)$ (b) \exitentropy curve $\xh^{(\ldegree)}(\ih)\longleftrightarrow\ih^{(\ldegree)}(\xh)$. The depicted ensembles are, in decreasing order, the $(100,200)$, the $(35,70)$, the $(12,24)$, the $(6,12)$, the $(4,8)$, the $(3,6)$ and the $(2,4)$ regular ensemble. While the BP threshold goes to 0, the bit MAP threshold goes to the Shannon limit $0.5$.
4988: }
4989: \label{fig:regcodes}
4990: \end{figure}
4991:
4992:
4993: \blemma \label{lemma:randomregularepsilon}
4994: For a fixed non-negative $\xl\in(0,1]$, denoting $\ih^{(\ldegree)}(\xl)\defas\frac{x}{(1-(1-\xl)^{\frac{\ldegree}{1-\drate}-1})^{\ldegree-1}}$, we get $\ih^{(\ldegree)}(\xl)\underset{\ldegree\to\infty}{\longrightarrow}\xl$.
4995: \elemma
4996: %\blemma \label{lemma:randomregularexit}
4997: %For a fixed non-negative $\xl\in(0,1]$, denoting $\xh^{(\ldegree)}(\xl)\defas(1-(1-\xl)^{\frac{\ldegree}{1-\drate}-1})^{\ldegree}$ we have $\xh^{(\ldegree)}(\xl)\underset{\ldegree\to\infty}{\longrightarrow}1$.
4998: %\elemma
4999: \bproof
5000: This limit is classically obtained with $(\ldegree-1)\log[1-(1-\xl)^{\frac{\ldegree}{1-\drate}-1}]\sim-(\ldegree-1)(1-\xl)^{\frac{\ldegree}{1-\drate}-1}$ which gives $(1-(1-\xl)^{\frac{\ldegree}{1-\drate}-1} )^{\ldegree-1} \underset{\ldegree\to\infty}{\longrightarrow}1^{-}$.
5001: \eproof
5002: \bprop\label{theo:randomregular}
5003: Consider the sequence $(x^{\ldegree-1},x^{\frac{\ldegree}{1-\drate}-1})\}_{\ldegree\geq2}$ with fixed
5004: rate $\drate<1$, then the BP threshold $\ih^\BP(\ldegree)\underset{\ldegree\to\infty}{\longrightarrow}0$.% whereas $\ih^\ML(\ldegree)\underset{\ldegree\to\infty}{\longrightarrow}\ih^{\Sh}=1-\drate>0$.
5005: \eprop
5006: \bproof
5007: Consider first the BP threshold $\ih^\BP(\ldegree)\defas\min_\xl\{\ih^{(\ldegree)}(\xl)\}$.
5008: Fix $\xi>0$ (very small). Clearly $0\leq\ih^\BP(\ldegree)\leq\ih^{(\ldegree)}(\frac{\xi}{2})$, and, since
5009: $\ih^{(\ldegree)}(\frac{\xi}{2})\underset{\ldegree\to\infty}{\longrightarrow}\frac{\xi}{2}$ with
5010: Lemma~\ref{lemma:randomregularepsilon}, we
5011: can state
5012: \[
5013: \exists \ldegree_0 \in \naturals, ~~ \forall \ldegree\geq\ldegree_0 ~~ \ih^{(\ldegree)}(\frac{\xi}{2})\leq \frac{\xi}{2}+\frac{\xi}{2}.
5014: \]
5015: This gives that, for all $\ldegree\geq\ldegree_0$, the statement $0\leq\ih^\BP(\ldegree)\leq \xi$ holds. This is true for any fixed $\xi$ meaning $\ih^\BP(\ldegree) \underset{\ldegree\to\infty}{\longrightarrow} 0$.
5016: \eproof
5017: Instead of studying the parameterized \exitentropy quantity $\xh(\xl)\defas(1-(1-\xl)^{\rdegree-1})^{\ldegree}$,
5018: it is often more convenient to work directly with the inverse mapping $\xh \mapsto \xl(\xh)\defas1-[1-\xh^{\frac{1}{\ldegree}}]^{\frac{1}{\rdegree-1}}$ such that we can eventually use $\ih(\xh)=\frac{1-[1-\xh^{\frac{1}{\ldegree}}]^{\frac{1}{\rdegree-1}}}{\xh^{\frac{\ldegree-1}{\ldegree}}}$ for $\xh\in(0,1]$.
5019: \blemma
5020: For a fixed $\xh\in(0,1)$, we have $\ih(\xh)=\frac{1-(1-\xh^{\frac{1}{\ldegree}})^{\frac{\drate-1}{\ldegree-\drate+1}}}{\xh^{\frac{\ldegree-1}{\ldegree}}} \underset{\ldegree\to\infty}{\longrightarrow}0.$
5021: \elemma
5022: \bproof
5023: The second term of the numerator goes to 1 since, $\log(1-\xh^{\frac{1}{\ldegree}})=\frac{\log\xh}{\ldegree}+\log(\frac{1}{\xh^{\frac{1}{\ldegree}}}-1)=\frac{\log\xh}{\ldegree} + \log(\frac{-\log\xh}{\ldegree}+o(\frac{1}{\ldegree}))$ such that ${ { \frac{\drate-1}{\ldegree-1+\drate}}} [ \frac{\log\xh}{\ldegree} + \log(\frac{-\log\xh}{\ldegree}+o(\frac{1}{\ldegree})) ] ~\underset{\ldegree\to\infty}{\longrightarrow}0$.
5024: %\[
5025: %{ { \frac{\drate-1}{\ldegree-1+\drate}}}\log(1-\xh^{\frac{1}{\ldegree}}) = { { \frac{\drate-1}{\ldegree-1+\drate}}}[\frac{\log\xh}{\ldegree}+\log(\frac{1}{\xh^{\frac{1}{\ldegree}}}-1)]
5026: %\]
5027: %\[
5028: %={ { \frac{\drate-1}{\ldegree-1+\drate}}} [ \frac{\log\xh}{\ldegree} + \log(\frac{-\log\xh}{\ldegree}+o(\frac{1}{\ldegree})) ] ~\underset{\ldegree\to\infty}{\longrightarrow}0,
5029: %\]
5030: The lemma follows from $\xh^{\frac{\ldegree-1}{\ldegree}}\sim\xh>0$.
5031: \eproof
5032: \bprop\label{theo:randomregularml}
5033: Consider again the sequence $(x^{\ldegree-1},x^{\frac{\ldegree}{1-\drate}-1})\}_{\ldegree\geq2}$ with fixed
5034: rate $\drate<1$, then $\ih^\MAP(\ldegree)\underset{\ldegree\to\infty}{\longrightarrow}\ih^{\Sh}=1-\drate>0$.
5035: \eprop
5036: \bproof
5037: First, the inequality $0\leq\ih^\Sh-\ih^\MAP(\ldegree)$ holds from the Area Theorem.\footnote{An alternative way is to show it via the Shannon Coding Theorem!}
5038: Second, \[\ih^\Sh-\ih^\MAP(\ldegree) = (1-\drate)-\ih^\MAP(\ldegree) = {\cal{A}}^{(\ldegree)} \leq {\tilde{\cal{A}}}^{(\ldegree)} \] where, in short, ${\cal{A}}^{(\ldegree)}$ represents the closed area between $\{\ih(\xh)\}_{\ih^\BP\leq\ih\leq1}$, the horizontal axis $\{\ih=\ih^\MAP\}$ and the vertical axis $\{\xh=1\}$. The area ${\tilde{\cal{A}}}^{(\ldegree)}$ is the surface of the unit square which lies under $\{\ih(\xh)\}_{0\leq\ih\leq1}$. Now, consider the function $\tilde{\ih}^{(\ldegree)}(\xh)=\min\{\ih(\xh)^{(\ldegree)},1\}\leq1$. The Dominated Convergence Theorem\footnote{Observe that $\ih(\xh)$ does not uniformly converge to $0$ on $(0,1)$ since $\int_0^1\ih(\xh)\text{d}\xh=1-\drate\neq0$.}
5039: applied to the sequence $\tilde{\ih}^{(\ldegree)}$ gives that
5040: $\lim_{\ldegree\to\infty}{\cal{\tilde{A}}}^{(\ldegree)} =0$, which concludes the proof.
5041: \eproof
5042: % It is easy to show the convergence of the two lemmas is uniform on
5043: % any [a,1] for a>0 since the max is found in a and ->0.
5044: % --> Link with hypothesis for Amin and the flattness cond.??
5045: \subsection{Application to other Iterative Coding Schemes}
5046: %{GLDPC and Turbo Ensembles} %"the Turbo corner"
5047: Although LDPC ensembles have been used to present the discussed concepts,
5048: the picture is not limited to such ensembles.
5049: Equivalent statements are expected to hold in large generality.
5050:
5051: To give just one example, consider generalized LDPC (GLDPC) ensembles:
5052: Part of our results can be directly applied like, e.g.,
5053: Lemma~\ref{theo:exactmlthreshold}.
5054: %\ref{theo:exactupperbound}. % to GLDPC ensembles.\label{theo:exactmlthreshold}
5055: Consider a GLDPC ensemble: Equivalently to the \ddp $(\ledge,\redge)$,
5056: the pair
5057: $(\ledge(\xl),\xr(\xl))\defas(\ledge(\xl),1-\redge(1-\xl))$
5058: suffices to describe the BP decoding of the ensemble in the asymptotic limit.
5059: The left (right) component of the pair $(\ledge(\xl),\xr(\xl))$ gives
5060: the \exitentropy entropy outgoing from the left (right) nodes during the BP decoding.
5061: To be more precise, at a fixed channel parameter $\ih$, %at a microscopic
5062: the function $\xl(\xr)\defas\ih\ledge(\xl)$ is the \exitentropy entropy outgoing from the
5063: left
5064: %\footnote{For Turbo codes, the degree distribution $\ledge\sum_{i\leq2}\ledge_i\xl^i$
5065: % represents only the systematic bits (edge perspective) since the parity bits are
5066: % simple variable nodes with degree 1.}
5067: and $\xr(\xl)\defas\xr(\xl,\ih)$ is the \exitentropy
5068: entropy outgoing from the right.\footnote{Contrary to the left nodes which stay
5069: simple repetition codes, the right nodes can be more complex linear codes.
5070: Therefore, $\xr(\xl)$ often depends on the edge type.
5071: For GLDPC ensembles, we consider
5072: the average over all types of node. For Turbo codes, one usually distinguish between systematic
5073: versus parity bits.}
5074: A few calculus or computations lead, in general, to an
5075: expression for the right component \exitentropy entropy (see, e.g, \cite{AKtB04,MeU03a}).
5076: \bex[GLDPC Codes]
5077: Generalized LDPC codes (see, e.g., \cite{Tan81,BPZ99,LZ98}) are LDPC codes whose check nodes are replaced by some
5078: more complex linear constraints. Such constraints are viewed as
5079: component codes which typically have minimum distance $d_{\text{min}}\geq 3$: they are bit MAP
5080: decoded and the component \exitentropy entropy $\xr(\xl)$ has smallest degree $d_{\text{min}}-1$ (see, e.g., \cite{MeU03a}).
5081: The \exitentropy entropy $\xr(\xl)$ is the function $ \xr(\xl)\defas\expectation\frac{1}{\rdegree}\sum_{i=1}^{\rdegree} \xr_i(\xl)$,
5082: where $\rdegree$ is the length of a particular component code and where the expectation is taken with respect to
5083: all such component codes. % (weighted according to the edge perspective)
5084: %goes through such component codes.
5085: The distribution $\ledge$ can be freely chosen but must satisfy the design rate constraint $\drate=1-\frac{1-\int \xr}{\int\ledge}$
5086: where $\int \xr$ is the rate of the average component code (Area Theorem). For example, consider GLDPC ensembles using
5087: $[2^p-1,2^p-p-1,3]$ binary Hamming codes as component codes. Then, when $\expectation d_{\text{min}}\geq3$, the BP \exitentropy entropy has
5088: at least one discontinuity at the BP threshold. It is given as,
5089: \[
5090: (\ih,\xh) =\Big(\frac{\xl}{\ledge(\xr(\xl))},\lnode(\xr(\xl))\Big).
5091: \]
5092: Theorem \ref{theo:exactmlthreshold} shows that, in general, $\ih^\BP\neq\ih^\MAP$ (The BP threshold being not given by the stability condition whenever the right component code has $d_{\text{min}}\geq 3$). In the next table, the first example uses $[7,4,3]$ Hamming codes such that its design rate is $\drate=\frac{1}{7}$ with the pair $(\ledge,\xr)=(\xl,3\xl^2 + 4\xl^3 - 15\xl^4 + 12\xl^5 - 3\xl^6)$ whereas the second example uses the $[15,11,3]$ Hamming code. It can be observed that this classical GLDPC have relatively bad BP threshold compared
5093: %\marginpar{good codes because only a few guesses might be needed if tightness in this case too.}
5094: to its MAP upper-bound. In the third example,
5095: $d_{\text{min}}$ is no longer $>2$ since we choose, in the node perspective,
5096: a mixture of 40 percent of $[7,6,2]$ Single Parity-Check codes,
5097: 40 percent of $[7,4,3]$ Hamming codes and 20 percent of $[15,11,3]$ Hamming codes.
5098: The BP \exitentropy function has however still a discontinuity at the BP threshold.
5099: \begin{center}
5100: {
5101: \tablespace
5102: \small{
5103: \begin{tabular}{ccccc}
5104: %\hline
5105: $\ledge(\xl)$ & $\xr(\xl)$ & $\ih^\BP$ & $\overline{\ih}^\MAP$ & $\ih^\Sh$ \\ \hline
5106: $\xl$ & $[7,4,3]$ & 0.75645 & 0.85616 & 0.85714 \\
5107: $\xl$ & $[15,11,3]$ & 0.46785 & 0.52780 & 0.53333 \\
5108: $\frac{3\xl+7\xl^8}{10}$ & mixture & 0.70483 & 0.71301 & 0.72801 \\
5109: \end{tabular}
5110: }
5111: \tablespace
5112: }
5113: \end{center}
5114: \eex
5115:
5116:
5117: \section{Conclusion}
5118:
5119: We have shown that there is
5120: a close connection between the BP and
5121: the MAP decoder. While this connection is quite general, we focused
5122: in this paper on communication over the binary erasure channel.
5123: In this case, the relation is furnished by the so-called Maxwell
5124: decoder which gives an operational meaning to the various areas
5125: under the EBP \exitentropy curve as number of guesses and number
5126: of confirmations.
5127: Unfortunately, this paper falls slightly short on several accounts
5128: of proving this relationship in the most general case. Let us
5129: summarize what seem to be the most important issues that still need
5130: to be addressed.
5131:
5132: %\begin{floatingfigure}{150bp}
5133: %\begin{figure}[hbt]
5134: %\vspace{10bp}
5135: %\centering
5136: %\setlength{\unitlength}{0.5bp}
5137: %\begin{picture}(225,225)
5138: %\put(0,0){\includegraphics[scale=0.7]{EXITconclusion}}
5139: %\end{picture}
5140: %\caption{\label{fig:exitconclusion} Generic Maxwell construction (See Example \ref{ex:running2jumps}).}
5141: %\vspace{5bp}
5142: %\end{figure}
5143: %\end{floatingfigure}
5144:
5145:
5146: First, there is currently no direct proof which establishes the
5147: existence of the asymptotic MAP \exitentropy curve. Rather, the existence
5148: follows from the explicit characterization of this limit. This occurs
5149: via Theorem \ref{CountingTheorem} in all those cases where
5150: the conditions of the theorem are fulfilled. Although theses conditions
5151: apply to a large class of ensembles, it would be pleasing to show
5152: the existence of the limit in the general case.
5153:
5154: A further point that needs some clarification is the restriction we
5155: had to impose in the second proof of Theorem \ref{theo:ebpareatheorem}.
5156: Recall that the argument on the computation tree via the Area Theorem
5157: required that the underlying ensemble has a non-trivial stability condition,
5158: since otherwise part of the EBP \exitentropy curve lies ``outside the unit box,''
5159: i.e., part of the curve corresponds to ``erasure probabilities above one.''
5160: While an analytical prove of Theorem \ref{theo:ebpareatheorem}
5161: is possible, it would be interesting (especially in view of generalizations)
5162: to have a conceptual proof valid for unconditionally stable ensembles.
5163: %For the BEC, this restriction is not too serious since the first proof
5164: %of Theorem \ref{theo:ebpareatheorem}, using calculus, applies always.
5165: %The picture changes though once we consider general channels.
5166: %In this setting, it is unlikely that one can find a proof equivalent
5167: %to the one using calculus, but the
5168: %argument on the support tree holds almost unchanged. It is
5169: %therefore of interest to see if the argument can be extended somewhat
5170: %to apply also to unconditionally stable ensembles.
5171:
5172: Without doubt the most important challenge is to assert the correctness of
5173: Conjecture \ref{conj:mainconjecture}. This would yield an easy and
5174: geometrically
5175: pleasing way of constructing the MAP \exitentropy curve from the EBP \exitentropy curve in
5176: the general case.
5177:
5178: Finally, an interesting research direction consists in the analysis
5179: of more general combinatorial search problems through a suitable
5180: `Maxwell construction'. An example (extremely close to the topic of this paper)
5181: consists in the problem of satisfiability of random sparse linear systems
5182: (`XORSAT')
5183: considered in ~\cite{MRTZ02,CDMM02}. The counting argument presented
5184: in Section \ref{sec:Counting} is indeed closely related to the approach
5185: of these papers. The ideas presented here can probably be used to analyze the
5186: behavior of simple resolution algorithms for this problem (see
5187: \cite{BLRZ02} for a numerical exploration).
5188: %
5189: %*******************************************************************
5190: %
5191: \section*{Acknowledgment}
5192: The authors would like to thank Nicolas Macris, Changyan Di, Gerhard Kramer,
5193: and Tom Richardson for useful discussions.
5194:
5195: A.M. has been partially supported by EVERGROW
5196: (i.p. 1935 in the
5197: complex systems initiative of the Future and Emerging Technologies
5198: directorate of the IST Priority, EU Sixth Framework).
5199:
5200: \appendices
5201: %
5202: \section{Proofs for Concentration Theorems} \label{app:concentrationproofs}
5203: Throughout this section, we use the shorthand $H_n =H_{\graph}(X|Y)$
5204: to denote the conditional entropy under transmission over the BMS channel
5205: $p_{Y_{[\n]} \mid X_{[\n]}}(\cdot \mid \cdot)$ using a code $\graph$
5206: chosen uniformly at random from $\ldpc(\n,\ledge,\redge)$.
5207: \label{app:ConcentrationProofs}
5208: %
5209: \subsection{Concentration of the Conditional Entropy}
5210: Fix an arbitrary order for the $m= (1-\rate)\n$ parity-check nodes, and let
5211: $\graph_t$, $t\in [m]$, be a random variable describing the first $t$ parity-check
5212: equations.
5213: %Explicitly, $\graph_t$ can be given by the first $t$ columns
5214: %of the parity-check matrix.
5215: Furthermore, let $\graph_0$ be a trivial (empty)
5216: random variable. Define the Doob martingale $Z_t \equiv \expectation[H_\n \mid \graph_t]$.
5217: The martingale property $\expectation[Z_{t+1} \mid Z_0, \dots, Z_t] = Z_t$ follows
5218: by construction.
5219: In order to stress that $Z_t$ is a (deterministic) function of the random variable
5220: $\graph_t$, we will write $Z_t = Z(\graph_t)$.
5221: Obviously, $Z_0 = \expectation[H_\n]$ is the expected conditional entropy over the
5222: code ensemble, and $Z_m = H_n \equiv H_{\graph}(X \mid Y)$ is the conditional entropy for a
5223: random code $\graph$. Theorem \ref{theo:concentrationml} follows therefore
5224: from the Hoeffding-Azuma inequality,
5225: once we bound the differences $|Z_{t+1}-Z_t|$. This is
5226: our aim in the remaining of this subsection.
5227:
5228: Assume, for the sake of definiteness, that parity-checks
5229: have been ordered by increasing
5230: degree. The first $m_1$ of them have degree $\rdegree_1$, the successive
5231: $m_2$ have degree $\rdegree_2$, and so on, with $\rdegree_1<\rdegree_2<\dots$.
5232: The $(t+1)^{\text{th}}$ parity-check will therefore have a well defined degree, to be denoted
5233: by $\rdegree$.
5234: Consider two realizations $\graph_{t+1}$ and $\graph'_{t+1}$
5235: of the first $(t+1)$ parity-checks which differ uniquely in
5236: the $(t+1)^{\text{th}}$ check.
5237: Let $\graph$ be a code uniformly distributed over $\ldpc(\ledge,\redge,\n)$
5238: whose restriction to the first $(t+1)$ parity-checks coincides with $\graph_{t+1}$.
5239: Construct a new code $\graph'$ whose restriction to the first $(t+1)$ parity-checks
5240: is $\graph'_{t+1}$, and which differs from $\graph$ in at most
5241: $(\rdegree+1)$ parity-checks. This can be done by the `switching' procedure of
5242: \cite{RiU01}. This switching procedure results in a ``pairing up'' of graphs.
5243: In order to obtain the desired result, it is now enough to show
5244: that $|H_{\graph}(X \mid Y)-H_{\graph'}(X \mid Y)|\le \alpha$,
5245: for some $n$-independent constant $\alpha$.
5246:
5247: Let us focus on the variation in conditional entropy under the addition
5248: of a single parity-check. Let $\graph$ be a generic linear code and let $\graph+1$,
5249: be the same code with the added constraint that
5250: $x_{i_1}\oplus \cdots\oplus x_{i_{\rdegree}}= \0$.
5251: Define the corresponding parity bit
5252: $\tilde{x}=x_{i_1}\oplus \cdots\oplus x_{i_{\rdegree}}$,
5253: Then
5254: %
5255: \begin{align*}
5256: H_{\graph}(X \mid Y) & = H_{\graph}(X \mid \tilde{X},Y)+
5257: H_{\graph}(\tilde{X} \mid Y) - H_{\graph}(\tilde{X} \mid X, Y) \\
5258: &= H_{\graph}(X \mid \tilde{X}=0,Y) + H_{\graph}(\tilde{X} \mid Y) \\
5259: &= H_{\graph+1}(X \mid Y) + H_{\graph}(\tilde{X} \mid Y) \, .
5260: \end{align*}
5261: %
5262: The second equality follows since
5263: $H_{\graph}(\tilde{X} \mid X, Y)=0$ and by using the channel symmetry.
5264: The third step is a consequence of the definition
5265: of $\graph+1$. Since $\tilde{X}$ is a bit, its entropy is between $0$ and $1$
5266: and therefore
5267: %
5268: \begin{align}
5269: %
5270: | H_{\graph}(X \mid Y) - H_{\graph+1}(X \mid Y)|\le 1\, .
5271: \label{eq:OneParityCheckVariation}
5272: %
5273: \end{align}
5274: %
5275:
5276: Recall that $\graph$ and $\graph'$ differ in at most ($\rdegree+1$)
5277: parity-checks, where $\rdegree$ is upper bounded by $\drmax$,
5278: the maximal check-node degree.
5279: Equation (\ref{eq:OneParityCheckVariation}) implies
5280: $|H_{\graph}(X \mid Y)-H_{\graph'}(X \mid Y)|\le (\rdegree+1)$
5281: and, therefore, Theorem~\ref{theo:concentrationml}.
5282: %
5283: %
5284: \subsection{Concentration of the Derivative of the Conditional Entropy}
5285: %
5286: It is convenient to introduce the per-bit conditional
5287: entropy $h_n(\cp) \defas \frac{1}{n}H_{\graph}(X|Y)$ and its expected
5288: value $\hb_n(\cp) \defas \frac{1}{n}\expectation H_{\graph}(X|Y)$ when $\graph$
5289: is a random code drawn uniformly from the $\ldpc(\ledge,\redge,\n)$ ensemble.
5290:
5291: Since the channel family $\{\text{BMS}(\cp)\}_{\cp \in I}$
5292: is smooth and ordered by physical degradation, $h_n(\cp)$ is
5293: differentiable convex function of $\cp\in I$. Therefore
5294: %
5295: \begin{align}
5296: %
5297: \frac{1}{\Delta}[h_n(\cp)-h_n(\cp-\Delta)]\le h'_n(\cp)\le
5298: \frac{1}{\Delta}[h_n(\cp+\Delta)-h_n(\cp)]\, ,\label{eq:ConvexDerivativeBound}
5299: %
5300: \end{align}
5301: %
5302: for any $\Delta>0$ such that $[\epsilon-\Delta,\epsilon+\Delta]\in I$.
5303: Because of Theorem \ref{theo:concentrationml}, we also have
5304: %
5305: \begin{align*}
5306: %
5307: \frac{1}{\Delta}[\hb_n(\cp)-\hb_n(\cp-\Delta)-2\tilde{\xi}]& \le h'_n(\cp)\le\\
5308: &\le \frac{1}{\Delta}[\hb_n(\cp+\Delta)-\hb_n(\cp)+2\tilde{\xi}]\, ,
5309: %
5310: \end{align*}
5311: %
5312: with probability greater than $1-Ae^{-nB\tilde{\xi}^2}$ (it follows from the proof
5313: in the previous subsection that $A$ and $B$ can be chosen uniformly in
5314: $\epsilon$). By averaging (\ref{eq:ConvexDerivativeBound})
5315: over the code $\graph$, and subtracting it from the last equation, we get
5316: %
5317: \begin{align*}
5318: %
5319: |h'_n(\cp)-\hb'_n(\cp)|\le \frac{1}{\Delta}
5320: [\hb_n(\cp\! +\!\Delta)-2\hb_n(\cp)+\hb_n(\cp\!-\!\Delta)+2\tilde{\xi}]\, ,
5321: %
5322: \end{align*}
5323: %
5324: which, using the convexity of $\hb_n(\epsilon)$, and fixing
5325: $\Delta = \tilde{\xi}^{1/2}$, implies
5326: %
5327: \begin{align*}
5328: %
5329: |h'_n(\cp)-\hb'_n(\cp)|\le [\hb_n'(\cp\! +\!\tilde{\xi}^{1/2})-
5330: \hb_n'(\cp\!-\!\tilde{\xi}^{1/2})]+2\tilde{\xi}^{1/2}\, .
5331: %
5332: \end{align*}
5333: %
5334: The functions $\hb_n$ are differentiable and convex and (by hypothesis)
5335: they converge to $\hb(\cp) = \xh^\MAP(\cp)=
5336: \lim_{\n\to\infty}\frac{1}{n}\expectation H_n$
5337: which is differentiable in $J$. It is a standard result in convex
5338: analysis (see \cite{Roc70}) that the derivatives $\hb'_n$ converge to $\hb'$
5339: uniformly in $J$. Therefore, there exists a sequence $\delta_n\to 0$,
5340: such that
5341: %
5342: \begin{align*}
5343: %
5344: |h'_n(\cp)-\hb'_n(\cp)|\le [\hb'(\cp\! +\!\tilde{\xi}^{1/2})-
5345: \hb'(\cp\!-\!\tilde{\xi}^{1/2})]+\delta_n+2\tilde{\xi}^{1/2}\, .
5346: %
5347: \end{align*}
5348: %
5349: with probability greater than $1-Ae^{-nB\tilde{\xi}^2}$. In order to complete
5350: the proof, it is sufficient to let $\tilde{\xi}_*(\xi)$ be the largest
5351: value of $\tilde{\xi}$, such that $[\hb'(\cp\! +\!\tilde{\xi}^{1/2})-
5352: \hb'(\cp\!-\!\tilde{\xi}^{1/2})]+2\tilde{\xi}^{1/2}<\xi/2$.
5353: Then the thesis holds with $\alpha_{\xi} = B\tilde{\xi}^2_*(\xi)/2$.
5354: In particular, if $\hb(\cp)$ is twice differentiable with respect to
5355: $\cp\in J$, then $[\hb'(\cp\! +\!\tilde{\xi}^{1/2})-
5356: \hb'(\cp\!-\!\tilde{\xi}^{1/2})]\le \tilde{A} \tilde{\xi}^{1/2}$,
5357: and $\tilde{\xi}_*(\xi)\ge \tilde{A}'\xi^2$.
5358: %
5359:
5360: %
5361: %*******************************************************************
5362: %
5363: \section{Proofs of Lemmas in the Counting Argument}
5364: \label{ProofCounting}
5365:
5366: %
5367: %*********************************
5368: %
5369: \subsection{Proof of Lemma \ref{ConcentrationDegreeLemma}}
5370: \label{ProofCountingDegree}
5371:
5372: Let $\graph(t)$ denote the residual graph after $t$ iterations
5373: of the message passing decoder, and
5374: $\nddp_{\graph(t)}=(\lnode_{\graph(t)},\Gamma_{\graph(t)})$ be the
5375: corresponding degree distribution pair. Moreover, denote
5376: by $\nddp_t = (\lnode_t,\Gamma_t)$ the typical degree
5377: distribution pair of $\graph(t)$. Explicitly
5378: %
5379: \begin{align*}
5380: %
5381: \lnode_t(\zl) & \defas \lnode(\zl \xl_t)\, ,\\
5382: \Gamma_{t}(\zl) &\defas \Gamma(1-\xr_t+\zl\xr_t)-\Gamma(1-\xr_t)-
5383: \zl\xr_t\Gamma'(1-\xr_t)\, ,
5384: %
5385: \end{align*}
5386: %
5387: where $\xl_{t},\xr_t$ denote the typical fractions of erased messages
5388: after $t$ iterations of the decoder. These are obtained by solving
5389: the density evolution equations $\xl_{t+1} = \cp\ledge(\xr_t)$,
5390: $\xr_{t+1}= 1-\redge(1-\xl_t)$ with initial condition $\xl_0=\xr_0=1$.
5391:
5392: Notice that
5393: %
5394: \begin{align*}
5395: %
5396: d(\nddp_{\cp},\nddp_{\graph{\cp}}) &\le d(\nddp_{\cp},\nddp_{t})
5397: +d(\nddp_{t},\nddp_{\graph(t)})+d(\nddp_{\graph(t)},\nddp_{\graph(\cp)})\, .
5398: %
5399: \end{align*}
5400: %
5401: We claim that
5402: %
5403: \begin{eqnarray}
5404: %
5405: \lim_{t\to\infty} d(\nddp_{\graph(t)},\nddp_{\graph(\cp)}) & = &0\, ,
5406: \label{FirstClaim}\\
5407: \lim_{n\to\infty} \expectation [d(\nddp_{t},\nddp_{\graph(t)})] & = & 0\, ,
5408: \label{SecondClaim}\\
5409: \lim_{t\to\infty}\lim_{n\to\infty} \expectation[ d(\nddp_{\cp},\nddp_{t})]
5410: & = & 0\, .\label{ThirdClaim}
5411: %
5412: \end{eqnarray}
5413: %
5414: Before proving those claims, let us show that they imply the thesis.
5415: It follows from the triangular inequality above that
5416: $\lim_{t\to\infty}\lim_{n\to\infty} \expectation \,
5417: d(\nddp_{\cp},\nddp_{\graph(\cp)})= 0$. But
5418: $d(\nddp_{\cp},\nddp_{\graph(\cp)})$ does not depend upon $t$, therefore
5419: %
5420: \begin{align*}
5421: %
5422: \lim_{n\to\infty} \expectation[ d(\nddp_{\cp},\nddp_{\graph(\cp)})] &= 0\, .
5423: %
5424: \end{align*}
5425: %
5426: This in turns imply the thesis via Markov inequality.
5427:
5428: We must now prove the inequalities (\ref{FirstClaim}) to (\ref{ThirdClaim}).
5429: The first one is a trivial consequence of the convergence of
5430: DE to its fixed point: $\lim_{t\to\infty}\xl_t=\xl$,
5431: $\lim_{t\to\infty}\xr_t=\xr$, together with the continuity of
5432: the expressions (\ref{LambdaRes}), (\ref{GammaRes}) with $\xl,\xr$.
5433: Eq. (\ref{SecondClaim}) follows from the general concentration
5434: analysis in \cite{RiU01}.
5435:
5436: In order to prove (\ref{ThirdClaim}), consider a variable node $i$
5437: in the residual graph and imagine changing the received symbol at $i$,
5438: and update all the messages consequently. Consider the edges whose distance
5439: from $i$ is larger than $t$, and denote by $W^{(t)}_i$ the number of
5440: messages on such edges that change of value after the received symbol at $i$
5441: has been changed. It is clear that
5442: %
5443: \begin{align}
5444: %
5445: \expectation[ d(\nddp_{\cp},\nddp_{t})] &\le \expectation[ W^{(t)}_i] ,
5446: %
5447: \end{align}
5448: %
5449: The limit $\lim_{n\to\infty} \expectation[W^{(t)}_i]$ can be computed
5450: through a branching process analysis. The calculation is very similar
5451: to the one in \cite{RiU00encoding} and we do not reproduce it here.
5452: The final result is that, as long as $\cp\ledge'(\xr)\redge'(1-\xl)<1$,
5453: there exist two positive constants $A$, $b$ with $b<1$ such that
5454: $\expectation [W^{(t)}_i ]\le A\, b^t$. The proof is finished by noticing
5455: that the condition $\cp\ledge'(\xr)\redge'(1-\xl)<1$ is satisfied whenever
5456: $\cp$ is a continuity point of $\xl(\cp)$.
5457: %
5458: %*********************************
5459: %
5460: \subsection{Proof of Lemma \ref{PsiProp}}
5461:
5462: Notice that the function $u\mapsto v(u)$ defined in (\ref{xtoy})
5463: enjoys the property $v(1/u) = 1/v(u)$ for any $u>0$.
5464: Assume {\em ab absurdum} that $\Psi_{\nddp}$ does not achieves
5465: its maximum in the interval $[0,1]$. Therefore, there exist $u>1$ such that
5466: $\Psi_{\nddp}(u')<\Psi_{\nddp}(u)$ for any $u'\in [0,1]$.
5467: We will show that $\Psi_{\nddp}(1/u)\ge \Psi_{\nddp}(u)$ thus
5468: reaching a contradiction.
5469: In fact, some algebra shows that
5470: %
5471: \begin{align*}
5472: %
5473: \Psi_{\nddp}(1/u) &= -\lnode'(1)\log_2\left[\frac{(1+uv)}{(1+u)(1+v)}\right]
5474: \\
5475: &~~ +\sum_\ldegree\lnode_\ldegree\log_2\left[
5476: \frac{1+u^\ldegree}{2(1+u)^\ldegree}\right]\\
5477: &~~ +\frac{\lnode'(1)}{\Gamma'(1)}\sum_{\rdegree}\Gamma_\rdegree
5478: \log_2\left[1+\left(\frac{v-1}{v+1}\right)^\rdegree\right].
5479: %
5480: \end{align*}
5481: %
5482: The claim follows from $0<\frac{v-1}{v+1}<1$ together with the monotonicity
5483: of the logarithm.
5484:
5485: In order to prove the second claim, i.e., the regularity of
5486: $\Psi_{\nddp}$ with respect to the \ddp write $\Psi_{\nddp}^{(1)}(u)
5487: +\Psi_{\nddp}^{(2)}(u)+\Psi_{\nddp}^{(3)}(u)$ with $\Psi^{(1,2,3)}_{\nddp}$
5488: the three summands in (\ref{Psidef}). The estimate (\ref{PsiLip})
5489: can be proved for each of the three terms separately.
5490: Here, we limit ourselves to consider $ \Psi_{\nddp}^{(1)}(u)$, the derivation
5491: being nearly identical for the two other summands.
5492: Start by noticing that, for any $u\in [0,1]$ and any \ddp, we have
5493: %
5494: \begin{eqnarray*}
5495: %
5496: \frac{1}{2}\le \sum_{\ldegree}\frac{\ledge_{\ldegree}}{1+u^{\ldegree}}\le 1\, ,
5497: \;\;\;\;\;\;\;\;\;\;\;
5498: \sum_{\ldegree}\frac{\ledge_{\ldegree}u^{\ldegree-1}}{1+u^{\ldegree}}\le 1\,.
5499: %
5500: \end{eqnarray*}
5501: %
5502: Now fix two \ddp $\nddp$ and $\tilde{\nddp}$. Let $v(u)$ and $\tilde{v}(u)$
5503: the corresponding functions defined as in (\ref{xtoy}).
5504: Notice that
5505: %
5506: \begin{align*}
5507: %
5508: \left|\sum_{\ldegree}\frac{\ledge_{\ldegree}-\ledge_{\ldegree}}
5509: {1+u^{\ldegree}}\right|
5510: &=\left|\sum_{\ldegree}\left(\frac{1}{1+u^{\ldegree}}-\frac{1}{2}\right)
5511: (\ledge_{\ldegree}-\ledge_{\ldegree})\right|\\
5512: &\le \frac{\ldegree_{\rm max}}{2}(1-u) \sum_{\ldegree}| \ledge_{\ldegree}-\ledge_{\ldegree}|\\
5513: &\le \frac{1}{2}\ldegree_{\rm max}^2(1-u)\,d(\nddp,\tilde{\nddp})
5514: %
5515: \end{align*}
5516: %
5517: Using these inequalities, some calculus shows that
5518: %
5519: \begin{align*}
5520: %
5521: 1\ge v(u), \tilde{v}(u) & \ge 1-2\, \ldegree_{\rm max}(1-u)\, ,\\
5522: |v(u)-\tilde{v}(u)| & \le 3\, \ldegree^2_{\rm max}\,\, (1-u)\,
5523: d(\nddp, \tilde{\nddp})\, .
5524: %
5525: \end{align*}
5526: %
5527: Next notice that, if we set $f(u,v) \defas \log_2\left[\frac{2(1+uv)}{(1+u)(1+v)}\right]$, then, for any $u,v,\tilde{v}\in [0,1]$, we have
5528: %
5529: \begin{align*}
5530: %
5531: \left| f(u,v)\right| & \le \frac{(1-u)(1-v)}{\log 2}\, ,\\
5532: \left|f(u,v)-f(u,\tilde{v})\right|
5533: &\le\frac{(1-u)}{\log 2}\, |v-\tilde{v}|\, .
5534: %
5535: \end{align*}
5536: %
5537: Using these observations we obtain
5538: %
5539: \begin{align*}
5540: %
5541: %&&\hspace{-1.cm}
5542: |\Psi_{\nddp}(u)-\Psi_{\tilde{\nddp}}(u)|
5543: & \le \max [f(u,v),f(u,\tilde{v})]
5544: \,|\lnode'(1)-\tilde{\lnode}'(1)| \\
5545: &~~ + \max[\lnode'(1),\tilde{\lnode}'(1)]
5546: \,|f(u,v)-f(u,\tilde{v})| \\
5547: &\le \frac{2\ldegree_{\rm max}}{\log 2}(1-u)^2|\lnode'(1)-\tilde{\lnode}'(1)|\\%+\\ %&& \;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;
5548: &~~ +\frac{\ldegree_{\rm max}}{\log 2}(1-u)|v-\tilde{v}| \\
5549: & \le A_1\, (1-u)^2\, d(\nddp,\tilde{\nddp})\, ,
5550: %
5551: \end{align*}
5552: %
5553: which confirms our thesis with constant $A_1 = (2\, \ldegree^2_{\rm max}+3
5554: \ldegree^3_{\rm max})/\log 2$. The variations of $\Psi^{(2)}_{\nddp}$
5555: and $\Psi^{(3)}_{\nddp}$ are bounded analogously.
5556: %
5557: %***************************************************************************
5558: %
5559:
5560: \section{Area and BP \exitentropy} \label{app:areass}
5561:
5562: \subsection{Two Useful Tricks}
5563: \label{sec:tricks}
5564: We give here two lemmas which contain the two computational tricks which
5565: are used all along this paper. Lemma~\ref{lemma:ibp1} and
5566: Lemma~\ref{lemma:ibp2} will be again used in the next subsection of the
5567: appendix.
5568: Observe that the function $\xl \mapsto \xh \defas \lnode(\xr(\xl))$ is composed by
5569: two functions $\xr$ and $\lnode$ which are strictly increasing over $[0,1]$. Therefore,
5570: the inverse function $\xl(\xh)$ exists and
5571: $\xh \mapsto \xl(\xh)\defas \xr^{-1} \circ \lnode^{-1}(\xh)$ is a continuous and strictly
5572: increasing bijection from $[0,1]$ to $[0,1]$. The values
5573: $\ih(\xl) \defas \frac{\xl}{\ledge(\xr(\xl))}$ can then equivalently
5574: be described by $\ih(\xh) \defas \frac{ \xr^{-1} \circ \lnode^{-1} }{\ledge\circ\lnode^{-1}}(\xh)$.
5575: %
5576: \blemma \label{lemma:ibp1} Given a \ddp $(\ledge,\redge)$ and any couple $(\xl_a,\xl_b)\in[0,1]^2$.
5577: With the notations
5578: $\xh_a=\xh(\xl_a)\defas \lnode\circ\xr(\xl_a)$ and $\xh_b=\xh(\xl_b)$, we can
5579: then write
5580: $$
5581: \int_{\xh_a}^{\xh_b} \ih(\xh)\text{d}\xh = \frac{1}{\int \ledge}\left(\xl_b \xr(\xl_b ) - \xl_a \xr(\xl_a ) -\int_{\xl_a}^{\xl_b}\xr(\xl)\text{d}\xl \right).
5582: $$
5583: \elemma
5584: \bproof This is a simple integration by parts once it has been observed
5585: $\ih(\xl)\cdot\frac{\text{d}\xh(\xl)}{\text{d}\xl}=\frac{\xl}{\ledge\circ\xr (\xl)}\cdot\frac{(\ledge\circ \xr)(\xl)\cdot\xr'(\xl)}{\int \ledge}=\frac{\xl \xr'(\xl)}{\int\ledge}$.
5586: \eproof
5587: %
5588: \blemma \label{lemma:ibp2} Given a \ddp $(\ledge,\redge)$ and any interval $(\xl_a\xl_b)\subseteq[0,1]$, $\xl^\BP\leq \xl_a$ over which $\ih(\xr)\defas\frac{\xl}{\ledge\circ\xr(\xl)}$ is increasing. Then, the function $\xh^\BP(\ih)$ is continuous over $(\ih_a, \ih_b)$, where $\ih_a\defas\ih(\xl_a)$ and $\ih_b\defas\ih(\xl_b)$, and
5589: \begin{multline*}
5590: \int_{\ih_a}^{\ih_b} \xh^\BP(\ih)\text{d}\ih = \frac{1}{\int \ledge}\Big( \ih_b \int_0^{\xr(\xl_b)}\ledge(\xr)\text{d}\xr - \ih_a \int_0^{\xr(\xl_a)}\ledge(\xr)\text{d}\xr\\ -\xl_b \xr(\xl_b ) + \xl_a \xr(\xl_a ) +\int_{\xl_a}^{\xl_b}\xr(\xl)\text{d}\xl \Big).
5591: \end{multline*}
5592: \elemma
5593: \bproof
5594: This is proved by, first, integrating by parts and, second, using Lemma~\ref{lemma:ibp1}.
5595: \eproof
5596:
5597:
5598: \subsection{Area under the BP \exitentropy Curve}
5599:
5600:
5601: %TODO: say this 2 lemmas are linked to microscopic analysis (dynamical view).
5602: \btheo[Area Theorem for BP Decoding]
5603: \label{theo:bpintegrationfact}
5604: Given a \ddp $(\ledge,\redge)$ and the asymptotic BP \exitentropy entropy as defined in Corollary~\ref{cor:asymptoticbpexit},
5605: then
5606: $$
5607: \drate+\frac{1}{\int \ledge}\sum_{i=1}^JD_i = \int_0^1 \xh^\BP(\ih)\text{d}\ih,
5608: $$
5609: where $D_i=A_i-B_i-C_i$ with $A_i\defas\lxl^{i} \xr(\lxl^{i})- \uxl^{i-1} \xr(\uxl^{i-1})$, $B_i\defas\ih^i \int_{\xr(\uxl^{i-1})}^{\xr(\lxl^i)} \ledge(\xr)\text{d}\xr$, and $C_i=\int_{\uxl^{i-1}}^{\lxl^i} \xr(\xl)\text{d}\xl
5610: $.
5611: \etheo
5612: %\bproof The proof is a direct application of Lemma~\ref{lemma:ibp2} and is explicited in Appendix~\ref{app:proofbpintegrationfact}.
5613: %\eproof
5614:
5615: \bproof
5616: Using Corollary~\ref{cor:asymptoticbpexit}, we can derive (\ref{eq:bpareatheo}) as shown above
5617: \begin{figure*}[hbt]
5618: \normalsize
5619: %\footnotesize
5620: \begin{align}
5621: &~ \notag \int_0^1 \xh^\BP(\ih)\text{d}\ih \\
5622: \notag &= \int_0^{\ih^\BP} \xh^\BP(\ih)\text{d}\ih + \sum_{i=1}^J\int_{\ih^i}^{\ih^{i+1}} \xh^\BP(\ih)\text{d}\ih \\ \notag
5623: & \overset{(a)}{=} 0 + \frac{1}{\int \ledge}\sum_{i=1}^J \left(\left[\ih(\xl)\int_0^{\xh(\xl)}\ledge(\xr)\text{d}\xr\right]_{\lxl^i}^{\uxl^i} - \Big[ \xl \xr(\xl) \Big]_{\lxl^i}^{\uxl^i} + \int_{\lxl^i}^{\uxl^i} \xr(\xl)\text{d}\xl \right)\\
5624: \notag &= \frac{
5625: \left(\int_0^1 \ledge(\xr)\text{d}\xr - \sum_{i=1}^J \left[\ih(\xl)\int_0^{\xh(\xl)}\ledge(\xr)\text{d}\xr \right]_{\uxl^{i-1}}^{\lxl^i} \right)
5626: - \left( 1 - \sum_{i=1}^J \Big[ \xl \xr(\xl) \Big]_{\uxl^{i-1}}^{\lxl^i} \right)
5627: + \left( \int_0^1 \xr(\xl)\text{d}\xl - \sum_{i=1}^J \int_{\uxl^{i-1}}^{\lxl^i} \xr(\xl)\text{d}\xl \right)
5628: }{\int \ledge}\\
5629: \label{eq:bpareatheo} & \overset{(b)}{=}\frac{\int \ledge -1 + \int \xr }{\ledge}+\frac{1}{\int \ledge}\sum_{i=1}^J
5630: \left(
5631: \Big[\xl \xr(\xl) \Big]_{\uxl^{i-1}}^{\lxl^i}
5632: - \ih^i \int_{\xr(\uxl^{i-1})}^{\xr(\lxl^i)} \ledge(\xr)\text{d}\xr
5633: - \int_{\uxl^{i-1}}^{\lxl^i} \xr(\xl)\text{d}\xl
5634: \right)
5635: \end{align}
5636: \hrulefill
5637: \vspace*{4pt}
5638: \end{figure*}
5639: where $(a)$ comes from Lemma~\ref{lemma:ibp2} and $(b)$ uses the fact that $\ih^i =\ih(\uxl^{i-1})=\ih(\lxl^i)$.
5640: \eproof
5641:
5642: First, observe that Theorem~\ref{theo:bpintegrationfact} quantifies the average
5643: sub-optimality of BP decoding compared to MAP decoding. The area under the BP
5644: \exitentropy curve is trivially larger or equal than the design rate since the
5645: $D_i$'s are non-negative. Moreover, it seems to indicate that there performance
5646: loss occurs at each phase transition.
5647:
5648: Second, Theorem~\ref{theo:bpintegrationfact} has a pleasing geometric interpretation
5649: which goes back to the asymptotic analysis and which is explained in
5650: appendix~\ref{app:gapinterpretation}.
5651:
5652:
5653:
5654:
5655:
5656: \section{Dynamic Interpretation of the Average Gap between MAP and BP Decoding}
5657: \label{app:gapinterpretation}
5658: It is now well-known that the determination of capacity-achieving sequences on the
5659: erasure channel reduces to
5660: a curve-fitting problem, see, e.g., \cite{Sho00}, \cite{AKtB04}. This was the motivation
5661: for the Area Theorem and - so far - its unique application. Let us recall this view.
5662: For the purpose of illustration, and without essential loss of generality,
5663: we focus on the case of (G)LDPC ensembles.
5664:
5665: \begin{figure}[htp]
5666: \centering
5667: \setlength{\unitlength}{1bp}
5668: \begin{picture}(230,70)
5669: \put(0,20){\includegraphics[scale=0.3125]{iterativeconv0}} %160*scale=50 [width=50bp]
5670: \put(60,20){\includegraphics[scale=0.3125]{iterativeconv1}}
5671: \put(120,20){\includegraphics[scale=0.3125]{iterativeconv2}}
5672: \put(180,20){\includegraphics[scale=0.3125]{iterativeconv3}}
5673: %
5674: \put(-4,15){\makebox(0,0){\footnotesize{$0$}}}
5675: \put(-4,67){\makebox(0,0){\footnotesize{$1$}}}
5676: \put(47,15){\makebox(0,0){\footnotesize{$1$}}}
5677: \put(25,5){\makebox(0,0){\footnotesize{$\ih=0.35<\ih^\BP$}}}
5678: %
5679: \put(60,0)
5680: {
5681: \put(-4,15){\makebox(0,0){\footnotesize{$0$}}}
5682: \put(-4,67){\makebox(0,0){\footnotesize{$1$}}}
5683: \put(47,15){\makebox(0,0){\footnotesize{$1$}}}
5684: \put(25,5){\makebox(0,0){\footnotesize{$\ih=0.43<\ih^\BP$}}}
5685: }
5686: %
5687: \put(120,0)
5688: {
5689: \put(-4,15){\makebox(0,0){\footnotesize{$0$}}}
5690: \put(-4,67){\makebox(0,0){\footnotesize{$1$}}}
5691: \put(47,15){\makebox(0,0){\footnotesize{$1$}}}
5692: \put(25,5){\makebox(0,0){\footnotesize{$\ih=0.58<\ih^\BP$}}}
5693: }
5694: %
5695: \put(180,0)
5696: {
5697: \put(-4,15){\makebox(0,0){\footnotesize{$0$}}}
5698: \put(-4,67){\makebox(0,0){\footnotesize{$1$}}}
5699: \put(47,15){\makebox(0,0){\footnotesize{$1$}}}
5700: \put(25,5){\makebox(0,0){\footnotesize{$\ih=0.61<\ih^\BP$}}}
5701: }
5702: \end{picture}
5703: \caption{Iterative decoding trajectory for the ensemble $\ldpc(\n,\xl^3,\xl^4)$ (in the limit when $\n\to\infty$):
5704: increasing values of the channel parameter $\ih$.}
5705: \label{fig:iterativetrajectory}
5706: \end{figure}
5707:
5708:
5709: \subsection{\exitentropy Chart}
5710: Fig.~\ref{fig:iterativetrajectory} summarizes the DE analysis of the BP decoding
5711: by showing the convergence of the recursive sequence formed the edge entropy $\{\xl_\iter\}_\iter$
5712: (i.e., the edge erasure probability).
5713: Such a representation (which emphasizes two component
5714: \exitentropy functions, one associated to the left nodes and one associated to the right nodes) is
5715: called \exitentropy chart in \cite{teB01}. This representation is
5716: (asymptotically)
5717: exact for the binary erasure channel (since it is DE) whereas it is only
5718: approximate in the general case.
5719:
5720:
5721: %\begin{floatingfigure}{120bp}
5722: \begin{figure}[htp]
5723:
5724: \vspace{10bp}
5725: \centering
5726: \setlength{\unitlength}{0.5bp}
5727: \begin{picture}(160,160)
5728: \put(0,0){\includegraphics[scale=0.5]{areatheoremflatness}}
5729: \put(-5,-5){\makebox(0,0){$0$}}
5730: \put(150,-5){\makebox(0,0){$1$}}
5731: \put(-5,150){\makebox(0,0){$1$}}
5732: \put(160, -10){\makebox(0,0){$\xl$}}
5733: \put(70, -10){\makebox(0,0){$\ih^\BP$}}
5734: \put(90, -10){\makebox(0,0){$\ih^\BP$}}
5735: \put(20, 80){\makebox(0,0){${\cal{D}}$}}
5736: %\put(96, 150){\makebox(0,0){${\cal{D}'}$}}
5737: \put(40, 143){\makebox(0,0){\footnotesize{ ${\ledge}^{-1}(\xl/\ih^\BP)$}}}
5738: \put(105, 105){\makebox(0,0){\footnotesize{ $\xr(\xl)\defas 1-\redge(1-\xl)$}}}
5739: \end{picture}
5740: \caption{ Additive gap to capacity for the \ddp $(x^3,x^4)$. }
5741: \label{fig:areatheoremflatness}
5742: \vspace{5bp}
5743: %\end{floatingfigure}
5744: \end{figure}
5745:
5746:
5747: Fig. \ref{fig:areatheoremflatness} represents the \exitentropy chart when transmission
5748: takes place at the BP threshold $\ih=\ih^\BP$. The \exitentropy functions are here
5749: the ones associated to the component of the LDPC ensemble. The function on
5750: the left is associated to repetition codes on the left while the one on the
5751: right is associated to parity-check codes. At channel parameter $\ih=\ih^\BP$,
5752: the two \exitentropy curves are tangent in $(\xl^\BP,\xr^\BP)$ and
5753: the \exitentropy chart offers also a graphical representation of the limiting gap to capacity of
5754: the LDPC ensemble. %Refer to Fig.~\ref{fig:areatheoremflatness}.
5755: The additive gap $C(\ih^\BP)-\drate$
5756: to the Shannon threshold is indeed represented by the entire white area ${\cal{D}}$
5757: such that
5758: $$
5759: C(\ih^\BP)-\drate=\ih^\Sh-\ih^\BP=\frac{{{\cal D }}}{\int\ledge},
5760: $$
5761: where $\frac{1}{\int \ledge}=\lnode'(1)$ is the average left degree. In words, the
5762: area ${\cal{D}}$ is the area between the left \exitentropy curve $\xl\mapsto \ledge^{-1}(\xl/\ih^\BP)$
5763: (at the BP threshold) and the right \exitentropy curve $\xl\mapsto1-\redge(1-\xl)$ which is bounded
5764: away by the unit square. This statement is presented, e.g., in \cite{AKtB04}.
5765: We will now refine this statement by applying the Area Theorem to
5766: the \exitentropy curve of the LDPC ensemble previous statement (i.e., using
5767: the basic
5768: principle of our method). We will see that, in short, the
5769: area ${\cal{D}}$ can be itself divided into two parts where the subarea below $\xl^\BP$
5770: represents the average gap between MAP and BP decoding. The determination of
5771: LDPC codes for which BP decoding is MAP reduces then again to a
5772: curve-fitting problem below $\xl^\BP$.
5773: %\marginpar{Link to the number of guesses.. .?}
5774:
5775: \subsection{Geometric Interpretation at the Component Level}\label{sec:geometryMAP-BP}
5776:
5777: Fig.~\ref{fig:geometricinterpretation} shows a geometric
5778: representation of Theorem~\ref{theo:bpintegrationfact}.
5779: In (a) one see that the additive gap between BP threshold and Shannon threshold
5780: is represented by the total area between the component \exitentropy functions.
5781: Further, the part of this area which
5782: corresponds to the average gap between MAP and BP decoding is $D_1$ as defined in Theorem~\ref{theo:bpintegrationfact}.
5783:
5784: \begin{figure}[htp]
5785: \centering
5786: \setlength{\unitlength}{1bp}
5787: \begin{picture}(240,140)
5788: \put(5,28)
5789: {
5790: \put(5,5){\includegraphics[width=100bp]{geometricinterpretation}}
5791: \put(135,5){\includegraphics[width=100bp]{geometricinterpretationJ}}
5792: % coordinates
5793: \put(0,0){\makebox(0,0){\small{$0$}}}
5794: \put(100,0){\makebox(0,0){\small{$1$}}}
5795: \put(0,100){\makebox(0,0){\small{$1$}}}
5796: \put(130,0){\makebox(0,0){\small{$0$}}}
5797: \put(230,0){\makebox(0,0){\small{$1$}}}
5798: \put(130,100){\makebox(0,0){\small{$1$}}}
5799: % left picture
5800: \put(7,-6){\makebox(0,0){\small{$\lxl^0$}}}
5801: \put(66,108){\makebox(0,0){\small{$\ih^\BP$}}}
5802: \put(18,80){\makebox(0,0){\small{$B_1$}}}
5803: \put(18,55){\makebox(0,0){\small{$D_1$}}}
5804: \put(38,40){\makebox(0,0){\small{$C_1$}}}
5805: \put(48,-6){\makebox(0,0){\small{$\lxl^1=\xl^\BP$}}}
5806: \put(37,100){\makebox(0,0){\small{$\ledge^{-1}(\frac{\xl}{\ih^\BP})$}}}
5807: \put(80,80){\makebox(0,0){\small{$\xr(\xl)\defas 1-\redge(1-\xl)$}}}
5808: % right picture
5809: \put(148,-5){\makebox(0,0){\small{$\xl^\BP$}}}%{\small{$\lxl^1=\xl^\BP$}}}
5810: \put(178,-5){\makebox(0,0){\small{$\uxl^1$}}}
5811: \put(208,-5){\makebox(0,0){\small{$\lxl^2$}}}
5812: \put(195,108){\makebox(0,0){\small{$\ih^\BP$}}}
5813: \put(220,108){\makebox(0,0){\small{$\ih^2$}}}
5814: \put(230,90){\makebox(0,0){\small{$\xr(\xl)$}}}
5815: \put(138,38){\makebox(0,0){\small{$B_1$}}}
5816: \put(142,09){\makebox(0,0){\small{$C_1$}}}
5817: \put(140,25){\makebox(0,0){\small{$D_1$}}}
5818: \put(158,80){\makebox(0,0){\small{$B_2$}}}
5819: \put(188,80){\makebox(0,0){\small{$D_2$}}}
5820: \put(188,40){\makebox(0,0){\small{$C_2$}}}
5821:
5822: }
5823: % caption
5824: \put(60,7){\makebox(0,0){{\small{(a)}}}}
5825: \put(190,7){\makebox(0,0){{\small{(b)}}}}
5826: \end{picture}
5827: \caption{
5828: Graphical interpretation of Theorem~\ref{theo:bpintegrationfact} at a microscopic (dynamic) level:
5829: (a) Standard one-jump case: Ensemble $\ldpc(x^3,x^4)$ and transmission at $\ih=\ih^\BP$
5830: (b) Double-jump case: The left distribution is
5831: $\ledge(x)=0.78x^2 + 0.1x^3 + 0.12x^{14}$ and the $\xr$ represents the
5832: \exitentropy function of a mixture of component codes composed by
5833: 50\% of $[19,18]$ single parity-check codes, 35\% of $[7,4]$ Hamming codes
5834: and 15\% of $[15,11]$ Hamming codes in the edge perspective. Transmission is represented
5835: for two channel parameters.
5836: }
5837: \label{fig:geometricinterpretation}
5838: \end{figure}
5839:
5840: % bibliography
5841: % ------------
5842: \bibliographystyle{IEEEtran} %plain
5843: %\bibliography{lth,lthpub}
5844:
5845: \newcommand{\SortNoop}[1]{}
5846: \begin{thebibliography}{10}
5847: \providecommand{\url}[1]{#1}
5848: \csname url@rmstyle\endcsname
5849: \providecommand{\newblock}{\relax}
5850: \providecommand{\bibinfo}[2]{#2}
5851: \providecommand\BIBentrySTDinterwordspacing{\spaceskip=0pt\relax}
5852: \providecommand\BIBentryALTinterwordstretchfactor{4}
5853: \providecommand\BIBentryALTinterwordspacing{\spaceskip=\fontdimen2\font plus
5854: \BIBentryALTinterwordstretchfactor\fontdimen3\font minus
5855: \fontdimen4\font\relax}
5856: \providecommand\BIBforeignlanguage[2]{{%
5857: \expandafter\ifx\csname l@#1\endcsname\relax
5858: \typeout{** WARNING: IEEEtran.bst: No hyphenation pattern has been}%
5859: \typeout{** loaded for the language `#1'. Using the pattern for}%
5860: \typeout{** the default language instead.}%
5861: \else
5862: \language=\csname l@#1\endcsname
5863: \fi
5864: #2}}
5865:
5866: \bibitem{MeU03b}
5867: C.~M{\'e}asson and R.~Urbanke, ``An upper-bound for the {ML} threshold of
5868: iterative coding systems over the {BEC},'' in \emph{Proc. of the Forty-First
5869: Allerton Conference}, Allerton House, Monticello, USA, October 1--3 2003.
5870:
5871: \bibitem{Mon01c}
5872: A.~Montanari, ``Why ``practical'' decoding algorithms are not as good as
5873: ``ideal'' ones?'' in \emph{Proc. DIMACS Workshop on Codes and Complexity},
5874: Rutgers University, Piscataway, USA, December 4--7 2001, pp. 63--66.
5875:
5876: \bibitem{MMU04}
5877: C.~M{\'e}asson, A.~Montanari, and R.~Urbanke, ``Maxwell's construction: The
5878: hidden bridge between maximum-likelihood and iterative decoding,'' in
5879: \emph{Proc. of the IEEE Int. Symposium on Inform. Theory}, Chicago, 2004.
5880:
5881: \bibitem{MMRU04}
5882: C.~M{\'e}asson, A.~Montanari, T.~Richardson, and R.~Urbanke, ``Life above
5883: threshold: From list decoding to area theorem and {MSE},'' in \emph{Proc. of
5884: the IEEE Inform. Theory Workshop}, San Antonio, Texas, October 24--29 2004.
5885:
5886: \bibitem{Per88}
5887: J.~Pearl, \emph{Probabilistic reasoning in intelligent systems: networks of
5888: plausible inference}.\hskip 1em plus 0.5em minus 0.4em\relax San Mateo:
5889: Morgan Kaufmann Publishers, 1988.
5890:
5891: \bibitem{Wib96}
5892: N.~Wiberg, ``Codes and decoding on general graphs,'' Ph.D. dissertation,
5893: Link{\"{o}}ping University, S-581 83, Link{\"{o}}ping, Sweden, 1996.
5894:
5895: \bibitem{AjM00}
5896: S.~M. Aji and R.~J. McEliece, ``The generalized distributive law,'' \emph{IEEE
5897: Trans. Inform. Theory}, vol.~46, no.~2, pp. 325--343, Mar. 2000.
5898:
5899: \bibitem{KFL01}
5900: F.~Kschischang, B.~Frey, and H.-A. Loeliger, ``Factor graphs and the
5901: sum-product algorithm,'' \emph{IEEE Transactions on Information Theory},
5902: vol.~47, no.~2, pp. 498--519, Feb. 2001.
5903:
5904: \bibitem{ETV99}
5905: T.~Etzion, A.~Trachtenberg, and A.~Vardy., ``Which codes have cycle-free
5906: {T}anner graphs?'' \emph{IEEE Trans. Inform. Theory}, vol.~45, no.~6, pp.
5907: 2173 -- 2181, Sept. 1999.
5908:
5909: \bibitem{RiU05}
5910: T.~Richardson and R.~Urbanke, \emph{Modern Coding Theory}.\hskip 1em plus 0.5em
5911: minus 0.4em\relax Cambridge University Press, 2005, in preparation.
5912:
5913: \bibitem{teB01}
5914: S.~ten Brink, ``Convergence behavior of iteratively decoded parallel
5915: concatenated codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~49, no.~10, pp.
5916: 1727--1737, Oct. 2001.
5917:
5918: \bibitem{AKtB02}
5919: A.~Ashikhmin, G.~Kramer, and S.~{ten Brink}, ``Code rate and the area under
5920: extrinsic information transfer curves,'' in \emph{Proc. of the IEEE Int.
5921: Symposium on Inform. Theory}, Lausanne, Switzerland, June 30--July 5 2002, p.
5922: 115.
5923:
5924: \bibitem{PiF04}
5925: H.~Pishro-Nik and F.~Fekri, ``On decoding of low-density parity-check codes
5926: over the binary erasure channel,'' \emph{IEEE Trans. Inform. Theory},
5927: vol.~50, no.~3, pp. 439--454, Mar. 2004.
5928:
5929: \bibitem{Gal63}
5930: R.~G. Gallager, \emph{Low-Density Parity-Check Codes}.\hskip 1em plus 0.5em
5931: minus 0.4em\relax Cambridge, Massachusetts: M.I.T. Press, 1963.
5932:
5933: \bibitem{LMSS01}
5934: M.~Luby, M.~Mitzenmacher, A.~Shokrollahi, and D.~A. Spielman, ``Efficient
5935: erasure correcting codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~47,
5936: no.~2, pp. 569--584, Feb. 2001.
5937:
5938: \bibitem{LMSS01b}
5939: ------, ``Improved low-density parity-check codes using irregular graphs,''
5940: \emph{IEEE Trans. Inform. Theory}, vol.~47, pp. 585--598, 2001.
5941:
5942: \bibitem{RiU01}
5943: T.~Richardson and R.~Urbanke, ``The capacity of low-density parity check codes
5944: under message-passing decoding,'' \emph{IEEE Trans. Inform. Theory}, vol.~47,
5945: pp. 599--618, Feb. 2001.
5946:
5947: \bibitem{RSU01}
5948: T.~Richardson, A.~Shokrollahi, and R.~Urbanke, ``Design of capacity-approaching
5949: irregular low-density parity-check codes,'' \emph{IEEE Trans. Inform.
5950: Theory}, vol.~47, pp. 619--637, Feb. 2001.
5951:
5952: \bibitem{Mon01}
5953: A.~Montanari, ``The glassy phase of {G}allager codes,'' \emph{European Physical
5954: Journal}, vol.~23, no. 121, 2001, arXiv:cond-math/cond-mat/0104079.
5955:
5956: \bibitem{BGT93}
5957: C.~Berrou, A.~Glavieux, and P.~Thitimajshima, ``Near {S}hannon limit
5958: error-correcting coding and decoding,'' in \emph{Proceedings of ICC'93},
5959: Geneve, Switzerland, May 1993, pp. 1064--1070.
5960:
5961: \bibitem{HOP96}
5962: J.~Hagenauer, E.~Offer, and L.~Papke, ``Iterative decoding of binary block and
5963: convolutional codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~42, pp.
5964: 429--445, Mar. 1996.
5965:
5966: \bibitem{MMRU05}
5967: C.~M{\'e}asson, A.~Montanari, T.~Richardson, and R.~Urbanke, ``The generalized
5968: area theorem and some of its consequences,'' 2005, in preparation.
5969:
5970: \bibitem{Mon04}
5971: A.~Montanari, ``Tight bounds for {LDPC} and {LDGM} codes under {MAP}
5972: decoding,'' \emph{IEEE Trans. Inform. Theory}, 2004, submitted.
5973:
5974: \bibitem{ZC95}
5975: G.~Z{\'e}mor and G.~Cohen, ``The threshold probability of a code,'' \emph{IEEE
5976: Trans. Inform. Theory}, vol.~41, pp. 469--477, Mar. 1995.
5977:
5978: \bibitem{BRU04}
5979: L.~Bazzi, T.~Richardson, and R.~Urbanke, ``Exact thresholds and optimal codes
5980: for the binary-symmetric channel and {G}allager's decoding algorithm {A},''
5981: \emph{IEEE Trans. Inform. Theory}, vol.~50, no.~9, Sept. 2004.
5982:
5983: \bibitem{Bol01}
5984: B.~Bollab\'{a}s, \emph{Random Graphs}.\hskip 1em plus 0.5em minus 0.4em\relax
5985: Cambridge University Press, 2001.
5986:
5987: \bibitem{DMU04}
5988: C.~Di, A.~Montanari, and R.~Urbanke, ``Weight distributions of {LDPC} code
5989: ensembles: combinatorics meets statistical physics,'' in \emph{Proc. of the
5990: IEEE Int. Symposium on Inform. Theory}, Chicago, USA, June 27--July 2 2004,
5991: p. 102.
5992:
5993: \bibitem{Gal62}
5994: R.~G. Gallager, ``Low-density parity-check codes,'' \emph{IRE Transactions on
5995: Information Theory}, Jan. 1962.
5996:
5997: \bibitem{MB04}
5998: G.~Miller and D.~Burshtein, ``Asymptotic enumeration method for analyzing
5999: {LDPC} codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~50, no.~6, pp.
6000: 1115--1131, June 2004.
6001:
6002: \bibitem{LS02}
6003: S.~L. Litsyn and V.~S. Shevelev, ``On ensembles of low-density parity-check
6004: codes: asymptotic distance distributions,'' \emph{IEEE Trans. Inform.
6005: Theory}, vol. IT--48, pp. 887 --908, Apr. 2002.
6006:
6007: \bibitem{LS03}
6008: ------, ``Distance distribution in ensembles of irregular low-density
6009: parity-check codes,'' \emph{IEEE Trans. Inform. Theory}, vol. IT--49, pp.
6010: 3140 --3159, Dec. 2003.
6011:
6012: \bibitem{DRU01}
6013: C.~Di, T.~Richardson, and R.~Urbanke, ``Weight distribution of iterative coding
6014: systems: How deviant can you be?'' in \emph{Proc. of the IEEE Int. Symposium
6015: on Inform. Theory}, Washington, USA, June 24--29 2001.
6016:
6017: \bibitem{DRU04}
6018: ------, ``Weight distribution of low-density parity-check codes,'' \emph{IEEE
6019: Trans. Inform. Theory}, 2004, submitted.
6020:
6021: \bibitem{Di04}
6022: C.~Di, ``Asymptotic and finite-length analysis of low-density parity-check
6023: codes,'' Ph.D. dissertation, EPFL, Lausanne, Switzerland, 2004, th{\`e}se no
6024: 3072.
6025:
6026: \bibitem{SeF96}
6027: R.~Sedgewick and P.~Flajolet, \emph{An Introduction to Analysis of
6028: Algorithms}.\hskip 1em plus 0.5em minus 0.4em\relax Addison-Wesley, 1996.
6029:
6030: \bibitem{FlS94}
6031: P.~Flajolet and R.~Sedgewick, ``The average case analysis of algorithms: Saddle
6032: point asymptotics,'' RR 2376, Tech. Rep., 1994.
6033:
6034: \bibitem{KiK02}
6035: C.~Kittel and H.~Kroemer, \emph{Thermal Physics}, 2nd~ed.\hskip 1em plus 0.5em
6036: minus 0.4em\relax New York: W. H. Freeman and Co., Mar. 2002.
6037:
6038: \bibitem{Mac97}
6039: D.~J.~C. MacKay, ``Good error correcting codes based on very sparse matrices,''
6040: \emph{IEEE Trans. Inform. Theory}, vol.~45, pp. 399--431, 1999.
6041:
6042: \bibitem{Andrea}
6043: A.~Montanari, ``The glassy phase of gallager codes,'' \emph{Eur. Phys. J. B},
6044: vol.~23, pp. 121--136, 2001.
6045:
6046: \bibitem{AKtB04}
6047: A.~Ashikhmin, G.~Kramer, and S.~ten Brink, ``Extrinsic information transfer
6048: functions: model and erasure channel property,'' \emph{IEEE Trans. Inform.
6049: Theory}, vol.~50, no.~11, pp. 2657--2673, Nov. 2004.
6050:
6051: \bibitem{MeU03a}
6052: C.~M{\'e}asson and R.~Urbanke, ``Further analytic properties of {EXIT}-like
6053: curves and applications,'' in \emph{Proc. of the IEEE Int. Symposium on
6054: Inform. Theory}, Yokohama, Japan, June 29--July 4 2003, p. 266.
6055:
6056: \bibitem{Tan81}
6057: R.~M. Tanner, ``A recursive approach to low complexity codes,'' \emph{IEEE
6058: Trans. Inform. Theory}, vol.~27, no.~5, pp. 533--547, Sept. 1981.
6059:
6060: \bibitem{BPZ99}
6061: J.~Boutros, O.~Pothier, and G.~Z{\'e}mor, ``Generalized low-density ({T}anner)
6062: codes,'' in \emph{Proceedings of the ICC'99}, Vancouver, Canada, June 1999,
6063: pp. 441-445.
6064:
6065: \bibitem{LZ98}
6066: M.~Lentmaier and K.~Zigangirov, ``Iterative decoding of generalized low-density
6067: parity-check codes,'' in \emph{Proc. of the IEEE Int. Symposium on Inform.
6068: Theory}, Boston, USA, August 16--21 1998, pp. 441-445.
6069:
6070: \bibitem{MRTZ02}
6071: M.~M\'ezard, F.~Ricci-Tersenghi, and R.~Zecchina, ``Alternative solutions to
6072: diluted p-spin models and {XORSAT} problems,'' \emph{J. Stat. Phys.}, vol.
6073: 111, p. 505, 2003, arXiv:cond-math/cond-mat/0207140.
6074:
6075: \bibitem{CDMM02}
6076: S.~Cocco, O.~Dubois, J.~Mandler, and R.~Monasson, ``Rigorous decimation-based
6077: construction of ground pure states for spin glass models on random
6078: lattices,'' \emph{Phys. Rev. Lett.}, vol.~90, no. 047205, 2003,
6079: arXiv:cond-math/cond-mat/0206239.
6080:
6081: \bibitem{BLRZ02}
6082: A.~Braunstein, M.~Leone, F.~Ricci-Tersenghi, and R.~Zecchina, ``Complexity
6083: transitions in global algorithms for sparse linear systems over finite
6084: fields,'' \emph{J. Stat. Phys.}, vol.~35, p. 7559, 2002,
6085: arXiv:cond-math/cond-mat/0203613.
6086:
6087: \bibitem{Roc70}
6088: R.~Rockafellar, \emph{Convex Analysis}.\hskip 1em plus 0.5em minus 0.4em\relax
6089: Princeton: Princeton University Press, 1970.
6090:
6091: \bibitem{RiU00encoding}
6092: T.~Richardson and R.~Urbanke, ``Efficient encoding of low-density parity-check
6093: codes,'' \emph{IEEE Trans. Inform. Theory}, vol.~47, pp. 638--656, Feb. 2001.
6094:
6095: \bibitem{Sho00}
6096: A.~Shokrollahi, ``Capacity-achieving sequences,'' in \emph{Codes, Systems, and
6097: Graphical Models}, ser. IMA Volumes in Mathematics and its Applications,
6098: B.~Marcus and J.~Rosenthal, Eds., vol. 123.\hskip 1em plus 0.5em minus
6099: 0.4em\relax Springer-Verlag, 2000, pp. 153--166.
6100:
6101: \end{thebibliography}
6102:
6103:
6104: \end{document}
6105:
6106:
6107: