I'm developing a Webapp (that includes Spring-data-hadoop) on a windows Box.
I installed Hadoop on a remote Ubuntu instance (in Amazon Ec2 cloud).
How Can I set the connection to Hadoop from my local Windows box?
I tried the following configuration, but I keep getting Connection refused exception (note that ${elasticIp} is the Amazon instance IP).
<hdp:configuration >
fs.default.name=hdfs://${elasticIp}:54310
hadoop.tmp.dir=/tmp
</hdp:configuration>
<hdp:job-runner id="myjob-runner" job-ref="myjob" run-at-startup="true" />
<hdp:job id="myjob" input-path="/tmp/input/" output-path="/tmp/output/"
mapper="org.apache.hadoop.examples.WordCount.Token izerMapper"
reducer="org.apache.hadoop.examples.WordCount.IntS umReducer" />
Stacktrace:
2012-11-11 10:44:37,442 ERROR - Context initialization failed [localhost-startStop-1] (org.springframework.web.context.ContextLoader:227 )
org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob-runner': Invocation of init method failed; nested exception is org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:192)
at org.springframework.beans.factory.support.DefaultL istableBeanFactory.preInstantiateSingletons(Defaul tListableBeanFactory.java:585)
at org.springframework.context.support.AbstractApplic ationContext.finishBeanFactoryInitialization(Abstr actApplicationContext.java:895)
at org.springframework.context.support.AbstractApplic ationContext.refresh(AbstractApplicationContext.ja va:425)
at org.springframework.web.context.ContextLoader.crea teWebApplicationContext(ContextLoader.java:282)
at org.springframework.web.context.ContextLoader.init WebApplicationContext(ContextLoader.java:204)
at org.springframework.web.context.ContextLoaderListe ner.contextInitialized(ContextLoaderListener.java: 47)
at org.apache.catalina.core.StandardContext.listenerS tart(StandardContext.java:4791)
at org.apache.catalina.core.StandardContext.startInte rnal(StandardContext.java:5285)
at org.apache.catalina.util.LifecycleBase.start(Lifec ycleBase.java:150)
at org.apache.catalina.core.ContainerBase.addChildInt ernal(ContainerBase.java:901)
at org.apache.catalina.core.ContainerBase.addChild(Co ntainerBase.java:877)
at org.apache.catalina.core.StandardHost.addChild(Sta ndardHost.java:618)
at org.apache.catalina.startup.HostConfig.deployDescr iptor(HostConfig.java:650)
at org.apache.catalina.startup.HostConfig$DeployDescr iptor.run(HostConfig.java:1582)
at java.util.concurrent.Executors$RunnableAdapter.cal l(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(Futu reTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.jav a:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.run Task(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:196)
at org.springframework.data.hadoop.mapreduce.JobExecu tor.findJobs(JobExecutor.java:91)
at org.springframework.data.hadoop.mapreduce.JobExecu tor.executeJobs(JobExecutor.java:64)
at org.springframework.data.hadoop.mapreduce.JobRunne r.call(JobRunner.java:52)
at org.springframework.data.hadoop.mapreduce.JobRunne r.afterPropertiesSet(JobRunner.java:44)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
... 26 more
Caused by: java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.apache.hadoop.ipc.Client.wrapException(Client. java:1099)
at org.apache.hadoop.ipc.Client.call(Client.java:1075 )
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java: 225)
at $Proxy59.getProtocolVersion(Unknown Source)
at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:396)
at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:379)
at org.apache.hadoop.hdfs.DFSClient.createRPCNamenode (DFSClient.java:119)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:238)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:203)
at org.apache.hadoop.hdfs.DistributedFileSystem.initi alize(DistributedFileSystem.java:89)
at org.apache.hadoop.fs.FileSystem.createFileSystem(F ileSystem.java:1386)
at org.apache.hadoop.fs.FileSystem.access$200(FileSys tem.java:66)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSyst em.java:1404)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:254)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:123)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:238)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java: 187)
at org.apache.hadoop.mapreduce.lib.input.FileInputFor mat.addInputPath(FileInputFormat.java:372)
at org.springframework.data.hadoop.mapreduce.JobFacto ryBean.afterPropertiesSet(JobFactoryBean.java:208)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
... 38 more
Caused by: java.net.ConnectException: Connection refused: no further information
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketC hannelImpl.java:567)
at org.apache.hadoop.net.SocketIOWithTimeout.connect( SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.ja va:489)
at org.apache.hadoop.ipc.Client$Connection.setupConne ction(Client.java:434)
at org.apache.hadoop.ipc.Client$Connection.setupIOstr eams(Client.java:560)
at org.apache.hadoop.ipc.Client$Connection.access$200 0(Client.java:184)
at org.apache.hadoop.ipc.Client.getConnection(Client. java:1206)
at org.apache.hadoop.ipc.Client.call(Client.java:1050 )
... 57 more
I installed Hadoop on a remote Ubuntu instance (in Amazon Ec2 cloud).
How Can I set the connection to Hadoop from my local Windows box?
I tried the following configuration, but I keep getting Connection refused exception (note that ${elasticIp} is the Amazon instance IP).
<hdp:configuration >
fs.default.name=hdfs://${elasticIp}:54310
hadoop.tmp.dir=/tmp
</hdp:configuration>
<hdp:job-runner id="myjob-runner" job-ref="myjob" run-at-startup="true" />
<hdp:job id="myjob" input-path="/tmp/input/" output-path="/tmp/output/"
mapper="org.apache.hadoop.examples.WordCount.Token izerMapper"
reducer="org.apache.hadoop.examples.WordCount.IntS umReducer" />
Stacktrace:
2012-11-11 10:44:37,442 ERROR - Context initialization failed [localhost-startStop-1] (org.springframework.web.context.ContextLoader:227 )
org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob-runner': Invocation of init method failed; nested exception is org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:192)
at org.springframework.beans.factory.support.DefaultL istableBeanFactory.preInstantiateSingletons(Defaul tListableBeanFactory.java:585)
at org.springframework.context.support.AbstractApplic ationContext.finishBeanFactoryInitialization(Abstr actApplicationContext.java:895)
at org.springframework.context.support.AbstractApplic ationContext.refresh(AbstractApplicationContext.ja va:425)
at org.springframework.web.context.ContextLoader.crea teWebApplicationContext(ContextLoader.java:282)
at org.springframework.web.context.ContextLoader.init WebApplicationContext(ContextLoader.java:204)
at org.springframework.web.context.ContextLoaderListe ner.contextInitialized(ContextLoaderListener.java: 47)
at org.apache.catalina.core.StandardContext.listenerS tart(StandardContext.java:4791)
at org.apache.catalina.core.StandardContext.startInte rnal(StandardContext.java:5285)
at org.apache.catalina.util.LifecycleBase.start(Lifec ycleBase.java:150)
at org.apache.catalina.core.ContainerBase.addChildInt ernal(ContainerBase.java:901)
at org.apache.catalina.core.ContainerBase.addChild(Co ntainerBase.java:877)
at org.apache.catalina.core.StandardHost.addChild(Sta ndardHost.java:618)
at org.apache.catalina.startup.HostConfig.deployDescr iptor(HostConfig.java:650)
at org.apache.catalina.startup.HostConfig$DeployDescr iptor.run(HostConfig.java:1582)
at java.util.concurrent.Executors$RunnableAdapter.cal l(Executors.java:441)
at java.util.concurrent.FutureTask$Sync.innerRun(Futu reTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.jav a:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.run Task(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.springframework.beans.factory.BeanCreationExce ption: Error creating bean with name 'myjob': Invocation of init method failed; nested exception is java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1422)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.doCreateBean(AbstractAu towireCapableBeanFactory.java:518)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.createBean(AbstractAuto wireCapableBeanFactory.java:455)
at org.springframework.beans.factory.support.Abstract BeanFactory$1.getObject(AbstractBeanFactory.java:2 93)
at org.springframework.beans.factory.support.DefaultS ingletonBeanRegistry.getSingleton(DefaultSingleton BeanRegistry.java:222)
at org.springframework.beans.factory.support.Abstract BeanFactory.doGetBean(AbstractBeanFactory.java:290 )
at org.springframework.beans.factory.support.Abstract BeanFactory.getBean(AbstractBeanFactory.java:196)
at org.springframework.data.hadoop.mapreduce.JobExecu tor.findJobs(JobExecutor.java:91)
at org.springframework.data.hadoop.mapreduce.JobExecu tor.executeJobs(JobExecutor.java:64)
at org.springframework.data.hadoop.mapreduce.JobRunne r.call(JobRunner.java:52)
at org.springframework.data.hadoop.mapreduce.JobRunne r.afterPropertiesSet(JobRunner.java:44)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
... 26 more
Caused by: java.net.ConnectException: Call to ec2-50-17-224-167.compute-1.amazonaws.com/50.17.224.167:54310 failed on connection exception: java.net.ConnectException: Connection refused: no further information
at org.apache.hadoop.ipc.Client.wrapException(Client. java:1099)
at org.apache.hadoop.ipc.Client.call(Client.java:1075 )
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java: 225)
at $Proxy59.getProtocolVersion(Unknown Source)
at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:396)
at org.apache.hadoop.ipc.RPC.getProxy(RPC.java:379)
at org.apache.hadoop.hdfs.DFSClient.createRPCNamenode (DFSClient.java:119)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:238)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient. java:203)
at org.apache.hadoop.hdfs.DistributedFileSystem.initi alize(DistributedFileSystem.java:89)
at org.apache.hadoop.fs.FileSystem.createFileSystem(F ileSystem.java:1386)
at org.apache.hadoop.fs.FileSystem.access$200(FileSys tem.java:66)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSyst em.java:1404)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:254)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:123)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.jav a:238)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java: 187)
at org.apache.hadoop.mapreduce.lib.input.FileInputFor mat.addInputPath(FileInputFormat.java:372)
at org.springframework.data.hadoop.mapreduce.JobFacto ryBean.afterPropertiesSet(JobFactoryBean.java:208)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.invokeInitMethods(Abstr actAutowireCapableBeanFactory.java:1479)
at org.springframework.beans.factory.support.Abstract AutowireCapableBeanFactory.initializeBean(Abstract AutowireCapableBeanFactory.java:1419)
... 38 more
Caused by: java.net.ConnectException: Connection refused: no further information
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketC hannelImpl.java:567)
at org.apache.hadoop.net.SocketIOWithTimeout.connect( SocketIOWithTimeout.java:206)
at org.apache.hadoop.net.NetUtils.connect(NetUtils.ja va:489)
at org.apache.hadoop.ipc.Client$Connection.setupConne ction(Client.java:434)
at org.apache.hadoop.ipc.Client$Connection.setupIOstr eams(Client.java:560)
at org.apache.hadoop.ipc.Client$Connection.access$200 0(Client.java:184)
at org.apache.hadoop.ipc.Client.getConnection(Client. java:1206)
at org.apache.hadoop.ipc.Client.call(Client.java:1050 )
... 57 more