1: \pdfoutput=1
2:
3: \documentclass[twoside,11pt]{article}
4:
5: % Any additional packages needed should be included after jmlr2e.
6: % Note that jmlr2e.sty includes epsfig, amssymb, natbib and graphicx,
7: % and defines many common macros, such as 'proof' and 'example'.
8: %
9: % It also sets the bibliographystyle to plainnat; for more information on
10: % natbib citation styles, see the natbib documentation, a copy of which
11: % is archived at http://www.jmlr.org/format/natbib.pdf
12:
13: \usepackage{jmlr2e}
14: \usepackage{amsmath}
15: \usepackage{url}
16:
17: %\usepackage{hyperref}
18: %\linespread{1.6}
19: % Definitions of handy macros can go here
20:
21: \newtheorem{thm}{Theorem}[section]
22: \newtheorem{cor}[thm]{Corollary}
23: \newtheorem{lem}[thm]{Lemma}
24: \newtheorem{qu}[thm]{Question}
25: \newtheorem{prop}[thm]{Proposition}
26: %\theoremstyle{remark}
27: \newtheorem{algo}[thm]{Algorithm}
28: %\theoremstyle{definition}
29: \newtheorem{defn}[thm]{Definition}
30: %\theoremstyle{remark}
31: \newtheorem{rem}[thm]{Remark}
32: %\numberwithin{equation}{section}
33:
34:
35:
36: %\linenumbers
37: \newcommand{\R}{\mathbb{R}}
38: \newcommand{\norm}[1]{\big\Vert#1\big\Vert}
39: \newcommand{\eps}{\varepsilon}
40: \newcommand{\argmax}{\operatornamewithlimits{argmax}}
41: \newcommand{\argmin}{\operatornamewithlimits{argmin}}
42: \newcommand{\one}{\textbf{1}}
43: \newcommand{\Xt}{\widetilde{X}_i}
44: \newcommand{\Tt}{\widetilde{T}}
45: \newcommand{\Lt}{\widetilde{L}}
46: \newcommand{\Ut}{\widetilde{U}}
47: \newcommand{\wt}{\tilde{w}_i}
48: \newcommand{\Pt}{\widetilde{P}}
49: \newcommand{\Uh}{\widehat{U}}
50: \newcommand{\li}{\lambda_1^i}
51:
52: % Heading arguments are {volume}{year}{pages}{submitted}{published}{author-full-names}
53:
54: %\jmlrheading{??}{??}{??}{??}{??}{??}
55:
56: % Short headings should be running head and authors last names
57:
58: \ShortHeadings{}{} \firstpageno{1}
59:
60: \begin{document}
61:
62: \title{LLE with low-dimensional neighborhood
63: representation}
64:
65: \author{\name Yair Goldberg \email yairgo@mail.huji.ac.il \\
66: \addr Department of Statistics\\
67: The Hebrew University, 91905 Jerusalem, Israel\\
68: \AND
69: \name Ya'acov Ritov \email yaacov.ritov@huji.ac.il \\
70: \addr Department of Statistics\\
71: The Hebrew University, 91905 Jerusalem, Israel\\
72: }
73:
74: \editor{??}
75:
76: \maketitle
77: %
78: \begin{abstract}% <- trailing '%' for backward compatibility of .sty file
79: The local linear embedding algorithm (LLE) is a non-linear
80: dimension-reducing technique, widely used due to its computational
81: simplicity and intuitive approach. LLE first linearly reconstructs
82: each input point from its nearest neighbors and then preserves these
83: neighborhood relations in the low-dimensional embedding. We show
84: that the reconstruction weights computed by LLE capture the
85: \emph{high}-dimensional structure of the neighborhoods, and not the
86: \emph{low}-dimensional manifold structure. Consequently, the weight
87: vectors are highly sensitive to noise. Moreover, this causes LLE to
88: converge to a \emph{linear} projection of the input, as opposed to
89: its \emph{non-linear} embedding goal. To overcome both of these
90: problems, we propose to compute the weight vectors using a
91: low-dimensional neighborhood representation. We prove theoretically
92: that this straightforward and computationally simple modification of
93: LLE reduces LLE's sensitivity to noise. This modification also
94: removes the need for regularization when the number of neighbors is
95: larger than the dimension of the input. We present numerical
96: examples demonstrating both the perturbation and linear projection
97: problems, and the improved outputs using the low-dimensional
98: neighborhood representation.
99: \end{abstract}
100:
101: \begin{keywords}
102: Locally Linear Embedding (LLE), dimension reduction , manifold
103: learning,
104: \end{keywords}
105:
106: \input{intro}
107: \input{description}
108: \input{highd_structure}
109: \input{localPCA}
110: \input{theory}
111: \input{numericalExamples}
112: \acks{This research was supported in part by Israeli Science
113: Foundation grant. Helpful discussions with Alon Zakai and Jacob
114: Goldberger are gratefully acknowledged.}
115:
116: \appendix
117: \input{proof1}
118: \input{proof2}
119:
120: % Acknowledgements should go at the end, before appendices and references
121:
122:
123:
124: \vskip 0.2in
125:
126:
127:
128: %\bibliography{lle_bib}
129: \begin{thebibliography}{26}
130: \expandafter\ifx\csname
131: natexlab\endcsname\relax\def\natexlab#1{#1}\fi
132: \expandafter\ifx\csname url\endcsname\relax
133: \def\url#1{{\tt #1}}\fi
134:
135: \bibitem[Balasubramanian et~al.(2002)Balasubramanian, Schwartz, Tenenbaum,
136: de~Silva, and Langford]{IsomapShortCircuits}
137: M.~Balasubramanian, E.~L. Schwartz, J.~B. Tenenbaum, V.~de~Silva,
138: and J.~C.
139: Langford.
140: \newblock The isomap algorithm and topological stability.
141: \newblock {\em Science}, 295\penalty0 (5552):\penalty0 7, 2002.
142:
143: \bibitem[Belkin and Niyogi(2003)]{belkin}
144: M.~Belkin and P.~Niyogi.
145: \newblock {Laplacian Eigenmaps for Dimensionality Reduction and Data
146: Representation}.
147: \newblock {\em Neural Comp.}, 15\penalty0 (6):\penalty0 1373--1396, 2003.
148:
149: \bibitem[Bernstein et~al.(2000)Bernstein, de~Silva, Langford, and
150: Tenenbaum]{IsoMapConvergence}
151: M.~Bernstein, V.~de~Silva, J.~C. Langford, and J.~B. Tenenbaum.
152: \newblock Graph approximations to geodesics on embedded manifolds.
153: \newblock Technical report, Stanford University, Stanford, Available at
154: \url{http://isomap.stanford.edu}, 2000.
155:
156: \bibitem[Chang and Yeung(2006)]{RLLE}
157: H.~Chang and D.~Y. Yeung.
158: \newblock Robust locally linear embedding.
159: \newblock {\em Pattern Recognition}, 39\penalty0 (6):\penalty0 1053--1065,
160: 2006.
161:
162: \bibitem[Chen et~al.(2007)Chen, Wang, Yan, Shan, Chen, and
163: Gao]{HumanFaceDetection}
164: J.~Chen, R.~Wang, S.~Yan, S.~Shan, X.~Chen, and W.~Gao.
165: \newblock Enhancing human face detection by resampling examples through
166: manifolds.
167: \newblock {\em Systems, Man and Cybernetics, Part A, IEEE Transactions on},
168: 37\penalty0 (6):\penalty0 1017--1028, 2007.
169:
170: \bibitem[Donoho and Grimes(2004)]{HessianEigenMap}
171: D.L. Donoho and C.~Grimes.
172: \newblock Hessian eigenmaps: Locally linear embedding techniques for
173: high-dimensional data.
174: \newblock {\em Proc. Natl. Acad. Sci. U.S.A.}, 100\penalty0 (10):\penalty0
175: 5591--5596, 2004.
176:
177: \bibitem[Goldberg et~al.(2008)Goldberg, Zakai, Kushnir, and
178: Ritov]{PriceOfNormalization}
179: Y.~Goldberg, A.~Zakai, D.~Kushnir, and Y.~Ritov.
180: \newblock Manifold learning: The price of normalization.
181: \newblock To appear in JMLR, 2008.
182:
183: \bibitem[Golub and Loan(1983)]{MatrixComputations}
184: G.~H. Golub and C.~F.~Van Loan.
185: \newblock {\em Matrix Computations}.
186: \newblock Johns Hopkins University Press, Baltimore, Maryland, 1983.
187:
188: \bibitem[Hadid and Pietik\"{a}inen(2003)]{ELLE}
189: A.~Hadid and M.~Pietik\"{a}inen.
190: \newblock Efficient locally linear embeddings of imperfect manifolds.
191: \newblock pages 188--201. 2003.
192:
193: \bibitem[Lee and Verleysen(2007)]{NLDR}
194: J.~A. Lee and M.~Verleysen.
195: \newblock {\em Nonlinear Dimensionality Reduction}.
196: \newblock Springer, 2007.
197:
198: \bibitem[L'Heureux et~al.(2004)L'Heureux, Carreau, Bengio, Delalleau, and
199: Yue]{Chemistry}
200: P.~J. L'Heureux, J.~Carreau, Y.~Bengio, O.~Delalleau, and S.~Y. Yue.
201: \newblock Locally linear embedding for dimensionality reduction in qsar.
202: \newblock {\em J. Comput. Aided Mol. Des.}, 18:\penalty0 475--482, 2004.
203:
204: \bibitem[Roweis and Saul(2000)]{LLE}
205: S.~T. Roweis and L.~K. Saul.
206: \newblock Nonlinear dimensionality reduction by locally linear embedding.
207: \newblock {\em Science}, 290\penalty0 (5500):\penalty0 2323--2326, 2000.
208:
209: \bibitem[Saul and Roweis()]{LLECode}
210: L.~K. Saul and S.~T. Roweis.
211: \newblock \textsc{L}ocally \textsc{L}inear \textsc{E}mbedding (\textsc{LLE})
212: homepage.
213: \newblock \url{http://www.cs.toronto.edu/~roweis/lle/}.
214:
215: \bibitem[Saul and Roweis(2003)]{thinkGlobally}
216: L.~K. Saul and S.~T. Roweis.
217: \newblock Think globally, fit locally: unsupervised learning of low dimensional
218: manifolds.
219: \newblock {\em J. Mach. Learn. Res.}, 4:\penalty0 119--155, 2003.
220: \newblock ISSN 1533-7928.
221:
222: \bibitem[Shi et~al.(2005)Shi, Shen, and Chen]{ImageDenoising}
223: R.~Shi, I.~F. Shen, and W.~Chen.
224: \newblock Image denoising through locally linear embedding.
225: \newblock In {\em CGIV '05: Proceedings of the International Conference on
226: Computer Graphics, Imaging and Visualization}, pages 147--152. IEEE Computer
227: Society, 2005.
228:
229: \bibitem[Tenenbaum et~al.()Tenenbaum, de~Silva, and Langford]{IsomapHompage}
230: J.~B. Tenenbaum, V.~de~Silva, and J.~C. Langford.
231: \newblock Isomap homepage.
232: \newblock \url{http://isomap.stanford.edu/}.
233:
234: \bibitem[Tenenbaum et~al.(2000)Tenenbaum, de~Silva, and Langford]{ISOMAP}
235: J.~B. Tenenbaum, V.~de~Silva, and J.~C. Langford.
236: \newblock A global geometric framework for nonlinear dimensionality reduction.
237: \newblock {\em Science}, 290\penalty0 (5500):\penalty0 2319--2323, 2000.
238:
239: \bibitem[Varini et~al.(2006)Varini, Degenhard, and Nattkemper]{ISOLLE}
240: C.~Varini, A.~Degenhard, and T.~W. Nattkemper.
241: \newblock Isolle: Lle with geodesic distance.
242: \newblock {\em Neurocomputing}, 69\penalty0 (13-15):\penalty0 1768--1771, 2006.
243:
244: \bibitem[Wang et~al.(2006)Wang, Zheng, Yao, and Li]{ILLE}
245: H.~Wang, J.~Zheng, Z.~Yao, and L.~Li.
246: \newblock Improved locally linear embedding through new distance computing.
247: \newblock pages 1326--1333. 2006.
248:
249: \bibitem[Wang et~al.(2005)Wang, Yang, Xu, and Chou]{Biology}
250: M.~Wang, H.~Yang, Z.~H. Xu, and K.~C. Chou.
251: \newblock Slle for predicting membrane protein types.
252: \newblock {\em J. Theor. Biol.}, 232\penalty0 (1):\penalty0 7--15, 2005.
253:
254: \bibitem[Weinberger and Saul(2006)]{Weinberger}
255: K.~Q. Weinberger and L.~K. Saul.
256: \newblock Unsupervised learning of image manifolds by semidefinite programming.
257: \newblock {\em International Journal of Computer Vision}, 70(1):\penalty0
258: 77--90, 2006.
259:
260: \bibitem[Wu and Hu(2006)]{LinearLLE}
261: F.~C. Wu and Z.~Y. Hu.
262: \newblock The \textsc{LLE} and a linear mapping.
263: \newblock {\em Pattern Recognition}, 39\penalty0 (9):\penalty0 1799--1804,
264: 2006.
265:
266: \bibitem[Xu et~al.(2008)Xu, Lifang, Dan, and Zhiyan]{SpeechVisualization}
267: W.~Xu, X.~Lifang, Y.~Dan, and H.~Zhiyan.
268: \newblock Speech visualization based on locally linear embedding (lle) for the
269: hearing impaired.
270: \newblock In {\em BMEI (2)}, pages 502--505, 2008.
271:
272: \bibitem[Xu et~al.(2006)Xu, Wu, Hu, and Luo]{Astrophysics}
273: X.~Xu, F.~C. Wu, Z.~Y. Hu, and A.~L. Luo.
274: \newblock A novel method for the determination of redshifts of normal galaxies
275: by non-linear dimensionality reduction.
276: \newblock {\em Spectroscopy and Spectral Analysis}, 26\penalty0 (1):\penalty0
277: 182--186, 2006.
278:
279: \bibitem[Zhang and Wang(2007)]{MLLE}
280: Z.~Zhang and J.~Wang.
281: \newblock Mlle: Modified locally linear embedding using multiple weights.
282: \newblock In B.~Sch\"{o}lkopf, J.~Platt, and T.~Hoffman, editors, {\em Advances
283: in Neural Information Processing Systems 19}, pages 1593--1600. MIT Press,
284: Cambridge, MA, 2007.
285:
286: \bibitem[Zhang and Zha(2004)]{LTSA}
287: Z.~Y. Zhang and H.~Y. Zha.
288: \newblock Principal manifolds and nonlinear dimensionality reduction via
289: tangent space alignment.
290: \newblock {\em SIAM J. Sci. Comp}, 26\penalty0 (1):\penalty0 313--338, 2004.
291:
292: \end{thebibliography}
293:
294: \end{document}
295: