1: \pdfoutput=1
2:
3: \documentclass{svjour3} % onecolumn (standard format)
4: %
5: \smartqed % flush right qed marks, e.g. at end of proof
6: %
7: \usepackage{graphicx}
8:
9: \usepackage{mathptmx} % use Times fonts if available on your TeX system
10: \usepackage{amsmath}
11: \usepackage{amsfonts}
12: \usepackage{natbib}
13: % insert here the call for the packages your document requires
14: \usepackage{latexsym}
15: %\usepackage{color}
16: % etc.
17: %
18: \newcommand{\norm}[1]{\left\Vert#1\right\Vert}
19: \newcommand{\abs}[1]{\left\vert#1\right\vert}
20: \newcommand{\set}[1]{\left\{#1\right\}}
21: \newcommand{\tr}[1]{\textrm{tr}\left(#1\right)}
22: \newcommand{\Real}{\mathbb{R}}
23: \newcommand{\R}{\mathbb{R}}
24: %\newcommand{\R}{R}
25: %\newcommand{\Real}{R}
26: \newcommand{\eps}{\varepsilon}
27: \newcommand{\To}{\longrightarrow}
28: \newcommand{\df}[1]{\frac{\partial}{\partial#1}}
29: \newcommand{\Hall}{H^{-}_{All}}
30: \newcommand{\argmax}{\operatornamewithlimits{argmax}}
31: \newcommand{\argmin}{\operatornamewithlimits{argmin}}
32: \newcommand{\OO}{\mathcal{O}}
33: \newcommand{\vp}{v_p^{\perp}}
34: \newcommand{\ap}{\alpha_p}
35: \newcommand{\bp}{\beta_p}
36: \newcommand{\Ji}{\hat{J_i}'}
37: \newcommand{\xj}{(x_{i_j}-\bar{x}_i)}
38: \newcommand{\zj}{(z_{i_j}-\bar{z}_i)}
39: \newcommand{\sjk}{\sum_{j=1}^{k(i)}}
40: \newcommand{\spd}{\sum_{p=1}^{d}}
41: \newcommand{\sjp}{\sum_{j,p=1}^{k(i),d}}
42: %\newcommand{\red}[1]{\textcolor{red}{#1}}
43:
44: % Insert the name of "your journal" with
45: \journalname{Machine Learning }
46: %
47: \begin{document}
48:
49: \title{Local Procrustes for Manifold Embedding:\\ A Measure of Embedding Quality and Embedding
50: Algorithms
51: \thanks{This research was supported in part by Israeli Science
52: Foundation grant.} }
53: % \subtitle{Do you have a subtitle?\\ If so,write it here}
54:
55: \titlerunning{Local Procrustes} % if too long for running head
56:
57: \author{Yair Goldberg \and
58: Ya'acov Ritov
59: }
60:
61: %\authorrunning{Short form of author list} % if too long for running head
62:
63: \institute{Y. Goldberg
64: \at Department of Statistics, The Hebrew University, 91905 Jerusalem, Israel\\
65: \email{yair.goldberg@mail.huji.ac.il} % \\
66: \and
67: Y. Ritov\\
68: \email{yaacov.ritov@huji.ac.il} }
69: %\at Department of Statistics, The Hebrew University, 91905 Jerusalem, Israel\\
70: \date{Received: date / Accepted: date}
71: % The correct dates will be entered by the editor
72:
73:
74: \maketitle
75:
76: \begin{abstract}
77: We present the Procrustes measure, a novel measure based on
78: Procrustes rotation that enables quantitative comparison of the
79: output of manifold-based embedding algorithms (such as
80: LLE~\citep{LLE} and Isomap~\citep{ISOMAP}). The measure also serves
81: as a natural tool when choosing dimension-reduction parameters. We
82: also present two novel dimension-reduction techniques that attempt
83: to minimize the suggested measure, and compare the results of these
84: techniques to the results of existing algorithms. Finally, we
85: suggest a simple iterative method that can be used to improve the
86: output of existing algorithms.
87:
88: \keywords{Dimension reducing \and Manifold learning \and Procrustes analysis, \and Local
89: PCA \and Simulated annealing}
90: % \PACS{PACS code1 \and PACS code2 \and more}
91: % \subclass{MSC code1 \and MSC code2 \and more}
92: \end{abstract}
93:
94:
95: \input{intro}
96: \input{problem}
97: \input{faithful}
98: \input{algo}
99: \input{numerical}
100: \input{discussion}
101: \appendix
102: \section{Proofs}
103:
104: \input{appendix}
105:
106:
107:
108: \begin{acknowledgements}
109: We would like to thank S. Kirkpatrick and J. Goldberger for
110: meaningful discussions. We are grateful to the anonymous reviewers
111: of an earlier version of this manuscript for their helpful
112: suggestions.
113: \end{acknowledgements}
114:
115:
116: \begin{thebibliography}{25}
117: \providecommand{\natexlab}[1]{#1} \providecommand{\url}[1]{{#1}}
118: \providecommand{\urlprefix}{URL } \expandafter\ifx\csname
119: urlstyle\endcsname\relax
120: \providecommand{\doi}[1]{DOI~\discretionary{}{}{}#1}\else
121: \providecommand{\doi}{DOI~\discretionary{}{}{}\begingroup
122: \urlstyle{rm}\Url}\fi
123: \providecommand{\eprint}[2][]{\url{#2}}
124:
125: \bibitem[{Belkin and Niyogi(2003)}]{belkin}
126: Belkin M, Niyogi P (2003) {Laplacian eigenmaps for dimensionality
127: reduction and
128: data representation}. Neural Comp 15(6):1373--1396
129:
130: \bibitem[{Bernstein et~al(2000.)Bernstein, de~Silva, Langford, and
131: Tenenbaum}]{IsoMapConvergence}
132: Bernstein M, de~Silva V, Langford JC, Tenenbaum JB (2000.) Graph
133: approximations
134: to geodesics on embedded manifolds, technical report, Stanford University,
135: Stanford, Available at http://isomap.stanford.edu
136:
137: \bibitem[{Cipra(1987)}]{IsingModel}
138: Cipra B (1987) An introduction to the \textsc{I}sing model. Am Math
139: Monthly
140: 94(10):937--959
141:
142: \bibitem[{Corman et~al(1990)Corman, Leiserson, and Rivest}]{corman}
143: Corman T, Leiserson C, Rivest R (1990) Introduction to Algorithms.
144: MIT Press
145:
146: \bibitem[{Dollar et~al(2007)Dollar, Rabaud, and Belongie}]{dollar}
147: Dollar P, Rabaud V, Belongie SJ (2007) Non-isometric manifold
148: learning:
149: analysis and an algorithm. In: Ghahramani Z (ed) Proceedings of the 24th
150: Annual International Conference on Machine Learning (ICML), Omnipress, pp
151: 241--248
152:
153: \bibitem[{Donoho and Grimes(2004)}]{HessianEigenMap}
154: Donoho D, Grimes C (2004) Hessian eigenmaps: Locally linear
155: embedding
156: techniques for high-dimensional data. Proc Natl Acad Sci USA
157: 100(10):5591--5596
158:
159: \bibitem[{Golub and Loan(1983)}]{MatrixComputations}
160: Golub GH, Loan CFV (1983) Matrix Computations. Johns Hopkins
161: University Press,
162: Baltimore, Maryland
163:
164: \bibitem[{Hull(1994)}]{twoDatabase}
165: Hull JJ (1994) A database for handwritten text recognition research.
166: IEEE Trans
167: Pattern Anal Mach Intell 16(5):550--554
168:
169: \bibitem[{Kirkpatrick et~al(1983)Kirkpatrick, Gelatt, and Vecchi}]{kirkpatrick}
170: Kirkpatrick S, Gelatt CD, Vecchi MP (1983) Optimization by simulated
171: annealing.
172: Science 220, 4598:671--680
173:
174: \bibitem[{Mardia et~al(1979)Mardia, Kent, and Bibby}]{Mardia}
175: Mardia K, Kent J, Bibby J (1979) Multivariate Analysis. Academic
176: Press
177:
178: \bibitem[{Munksgaard(1980)}]{SparseMatrices}
179: Munksgaard N (1980) Solving sparse symmetric sets of linear
180: equations by
181: preconditioned conjugate gradients. ACM Trans Math Softw 6(2):206--219
182:
183: \bibitem[{Roweis(retrieved Nov. 2006)}]{FreyFace}
184: Roweis S (retrieved Nov. 2006) Frey face on sam roweis' page,
185: \url{http://www.cs.toronto.edu/~roweis/data.html}
186:
187: \bibitem[{Roweis and Saul(2000)}]{LLE}
188: Roweis ST, Saul LK (2000) {Nonlinear dimensionality reduction by
189: locally linear
190: embedding}. Science 290(5500):2323--2326
191:
192: \bibitem[{Roweis et~al(2001)Roweis, Saul, and
193: Hinton}]{ProbabilisticGlobalCoordinationRoweis}
194: Roweis ST, Saul LK, Hinton GE (2001) Global coordination of local
195: linear
196: models. In: Advances in Neural Information Processing Systems 14, MIT Press,
197: pp 889--896
198:
199: \bibitem[{Sha and Saul(2005)}]{ShaExtensionSpectralMethods}
200: Sha F, Saul LK (2005) Analysis and extension of spectral methods for
201: nonlinear
202: dimensionality reduction. In: Machine Learning, Proceedings of the
203: Twenty-Second International Conference (ICML), pp 784--791
204:
205: \bibitem[{Siarry et~al(1997)Siarry, Berthiau, Durdin, and
206: Haussy}]{SA_continuous}
207: Siarry P, Berthiau G, Durdin F, Haussy J (1997) Enhanced simulated
208: annealing
209: for globally minimizing functions of many-continuous variables. ACM Trans
210: Math Softw 23(2):209--228
211:
212: \bibitem[{Sibson(1978)}]{sibson}
213: Sibson R (1978) Studies in robustness of multidimensional-scaling:
214: Procrustes
215: statistics. J Roy Statist Soc 40(2):234--238
216:
217: \bibitem[{Sibson(1979)}]{SibsonRobustness}
218: Sibson R (1979) Studies in the robustness of
219: multidimensional-scaling:
220: Perturbational analysis of classical scaling. J Roy Statist Soc
221: 41(2):217--229
222:
223: \bibitem[{de~Silva and Tenenbaum(2003)}]{cIsomap}
224: de~Silva V, Tenenbaum JB (2003) Global versus local methods in
225: nonlinear
226: dimensionality reduction. In: Advances in Neural Information Processing
227: Systems 15, MIT Press
228:
229: \bibitem[{Stewart(1980)}]{generatingRandomMatrices}
230: Stewart GW (1980) The efficient generation of random orthogonal
231: matrices with
232: an application to condition estimators. SIAM Journal on Numerical Analysis
233: 17(3):403--409
234:
235: \bibitem[{Teh and Roweis(2003)}]{TehAutomaticAlignment}
236: Teh YW, Roweis S (2003) Automatic alignment of local
237: representations. In:
238: Becker S, Thrun S, Obermayer K (eds) Advances in Neural Information
239: Processing Systems 15, MIT Press
240:
241: \bibitem[{Tenenbaum et~al(2000)Tenenbaum, de~Silva, and Langford}]{ISOMAP}
242: Tenenbaum JB, de~Silva V, Langford JC (2000) A global geometric
243: framework for
244: nonlinear dimensionality reduction. Science 290(5500):2319--2323
245:
246: \bibitem[{Verbeek et~al(2002)Verbeek, Vlassis, and
247: Kr\"{o}se}]{VerbeekCoordinatingPCA}
248: Verbeek J, Vlassis N, Kr\"{o}se B (2002) Coordinating principal
249: component
250: analyzers. In: Proceedings of International Conference on Artificial Neural
251: Networks
252:
253: \bibitem[{Weinberger and Saul(2006)}]{Weinberger}
254: Weinberger K, Saul L (2006) Unsupervised learning of image manifolds
255: by
256: semidefinite programming. Int J Comput Vision 70(1):77--90
257:
258: \bibitem[{Zhang and Zha(2004)}]{LTSA}
259: Zhang Z, Zha H (2004) Principal manifolds and nonlinear
260: dimensionality
261: reduction via tangent space alignment. SIAM J Sci Comp 26(1):313--338
262:
263: \end{thebibliography}
264:
265: % BibTeX users please use one of
266: %\bibliographystyle{spbasic} % basic style, author-year citations
267: %\bibliographystyle{spmpsci} % mathematics and physical sciences
268: %\bibliographystyle{spphys} % APS-like style for physics
269: %\bibliography{lpr} % name your BibTeX data base
270:
271:
272: \end{document}
273: