Skip to content

Commit 964a08c

Browse files
committed
Merge pull request #48 from childsb/hadoop2
hadoop 2.0 impl
2 parents abffec9 + ce63253 commit 964a08c

File tree

6 files changed

+261
-8
lines changed

6 files changed

+261
-8
lines changed

conf/core-site.xml

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,16 @@
1212

1313
<property>
1414
<name>fs.default.name</name>
15-
<value>glusterfs://RHS-1:9000</value>
15+
<value>glusterfs://ambari-3.abrv8.com:gv0<</value>
1616
</property>
1717

18+
<property>
19+
<name>fs.AbstractFileSystem.glusterfs.impl</name>
20+
<value>org.apache.hadoop.fs.local.GlusterFs</value>
21+
</property>
22+
23+
24+
1825
<property>
1926
<name>fs.glusterfs.volname</name>
2027
<value>HadoopVol</value>
@@ -30,8 +37,6 @@
3037
<value>RHS-1</value>
3138
</property>
3239

33-
<property>
34-
<name>fs.glusterfs.write.buffer.size</name>
35-
<value>1024</value>
36-
</property>
40+
41+
3742
</configuration>

conf/mapred-site.xml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<!--Sun Jun 9 13:00:26 2013-->
2+
<configuration>
3+
<property>
4+
<name>yarn.app.mapreduce.am.staging-dir</name>
5+
<value>glusterfs:///tmp/hadoop-yarn/staging/mapred/.staging</value>
6+
</property>
7+
<property>
8+
<name>mapred.healthChecker.script.path</name>
9+
<value>glusterfs:///mapred/jobstatus</value>
10+
</property>
11+
<property>
12+
<name>mapred.job.tracker.history.completed.location</name>
13+
<value>glusterfs:///mapred/history/done</value>
14+
</property>
15+
<property>
16+
<name>mapred.system.dir</name>
17+
<value>glusterfs:///mapred/system</value>
18+
</property>
19+
<property>
20+
<name>mapreduce.jobtracker.staging.root.dir</name>
21+
<value>glusterfs:///user</value>
22+
</property>
23+
</configuration>

pom.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414
<version>4.4</version>
1515
</dependency>
1616
<dependency>
17-
<groupId>org.apache.hadoop</groupId>
18-
<artifactId>hadoop-core</artifactId>
19-
<version>0.20.2</version>
17+
<groupId>org.apache.hadoop</groupId>
18+
<artifactId>hadoop-common</artifactId>
19+
<version>2.0.5-alpha</version>
2020
</dependency>
2121
<dependency>
2222
<groupId>org.apache.hadoop</groupId>
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package org.apache.hadoop.fs.local;
2+
3+
import java.io.IOException;
4+
import java.net.URI;
5+
import java.net.URISyntaxException;
6+
7+
import org.apache.hadoop.conf.Configuration;
8+
import org.apache.hadoop.fs.ChecksumFs;
9+
10+
public class GlusterFs extends ChecksumFs{
11+
12+
GlusterFs(Configuration conf) throws IOException, URISyntaxException{
13+
super(new GlusterVol(conf));
14+
}
15+
16+
GlusterFs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException{
17+
this(conf);
18+
}
19+
20+
}
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package org.apache.hadoop.fs.local;
2+
3+
import java.io.IOException;
4+
import java.net.URI;
5+
import java.net.URISyntaxException;
6+
7+
import org.apache.hadoop.conf.Configuration;
8+
import org.apache.hadoop.fs.FsConstants;
9+
import org.apache.hadoop.fs.Path;
10+
import org.apache.hadoop.fs.RawLocalFileSystem;
11+
import org.apache.hadoop.fs.glusterfs.GlusterFileSystem;
12+
import org.apache.hadoop.fs.glusterfs.GlusterVolume;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
public class GlusterVol extends RawLocalFsG{
17+
18+
protected String glusterMount = null;
19+
protected static final Logger log = LoggerFactory.getLogger(GlusterFileSystem.class);
20+
21+
22+
GlusterVol(final Configuration conf) throws IOException, URISyntaxException {
23+
this(GlusterVolume.NAME, conf);
24+
25+
}
26+
27+
/**
28+
* This constructor has the signature needed by
29+
* {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
30+
*
31+
* @param theUri which must be that of localFs
32+
* @param conf
33+
* @throws IOException
34+
* @throws URISyntaxException
35+
*/
36+
GlusterVol(final URI theUri, final Configuration conf) throws IOException, URISyntaxException {
37+
super(theUri, new GlusterVolume(), conf, false);
38+
}
39+
40+
41+
42+
43+
44+
}
Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
package org.apache.hadoop.fs.local;
2+
3+
import java.io.FileNotFoundException;
4+
import java.io.IOException;
5+
import java.net.URI;
6+
import java.net.URISyntaxException;
7+
8+
import org.apache.hadoop.conf.Configuration;
9+
import org.apache.hadoop.fs.DelegateToFileSystem;
10+
import org.apache.hadoop.fs.FileStatus;
11+
import org.apache.hadoop.fs.FileSystem;
12+
import org.apache.hadoop.fs.FsConstants;
13+
import org.apache.hadoop.fs.FsServerDefaults;
14+
import org.apache.hadoop.fs.Path;
15+
import org.apache.hadoop.fs.RawLocalFileSystem;
16+
import org.apache.hadoop.fs.permission.FsPermission;
17+
import org.apache.hadoop.util.Shell;
18+
19+
public class RawLocalFsG extends DelegateToFileSystem{
20+
21+
/*
22+
* need a constructor on apache's RawLocalFileSystem that takes FileSystem object
23+
*
24+
*/
25+
26+
RawLocalFsG(final URI theUri, final FileSystem fs, final Configuration conf,boolean myMamaFat) throws IOException, URISyntaxException{
27+
super(theUri, fs, conf, theUri.getScheme(), myMamaFat);
28+
}
29+
30+
31+
32+
/* ------------------- below this line is c + p from RawLocalFs.. --------------------------*/
33+
RawLocalFsG(final Configuration conf) throws IOException, URISyntaxException {
34+
this(FsConstants.LOCAL_FS_URI, conf);
35+
}
36+
37+
/**
38+
* This constructor has the signature needed by
39+
* {@link AbstractFileSystem#createFileSystem(URI, Configuration)}.
40+
*
41+
* @param theUri which must be that of localFs
42+
* @param conf
43+
* @throws IOException
44+
* @throws URISyntaxException
45+
*/
46+
RawLocalFsG(final URI theUri, final Configuration conf) throws IOException,
47+
URISyntaxException {
48+
super(theUri, new RawLocalFileSystem(), conf,
49+
FsConstants.LOCAL_FS_URI.getScheme(), false);
50+
}
51+
52+
53+
@Override
54+
public int getUriDefaultPort() {
55+
return -1; // No default port for file:///
56+
}
57+
58+
@Override
59+
public FsServerDefaults getServerDefaults() throws IOException {
60+
return LocalConfigKeys.getServerDefaults();
61+
}
62+
63+
@Override
64+
public boolean supportsSymlinks() {
65+
return true;
66+
}
67+
68+
@Override
69+
public void createSymlink(Path target, Path link, boolean createParent)
70+
throws IOException {
71+
final String targetScheme = target.toUri().getScheme();
72+
if (targetScheme != null && !"file".equals(targetScheme)) {
73+
throw new IOException("Unable to create symlink to non-local file "+
74+
"system: "+target.toString());
75+
}
76+
if (createParent) {
77+
mkdir(link.getParent(), FsPermission.getDirDefault(), true);
78+
}
79+
// NB: Use createSymbolicLink in java.nio.file.Path once available
80+
try {
81+
Shell.execCommand(Shell.LINK_COMMAND, "-s",
82+
new URI(target.toString()).getPath(),
83+
new URI(link.toString()).getPath());
84+
} catch (URISyntaxException x) {
85+
throw new IOException("Invalid symlink path: "+x.getMessage());
86+
} catch (IOException x) {
87+
throw new IOException("Unable to create symlink: "+x.getMessage());
88+
}
89+
}
90+
91+
/**
92+
* Returns the target of the given symlink. Returns the empty string if
93+
* the given path does not refer to a symlink or there is an error
94+
* acessing the symlink.
95+
*/
96+
private String readLink(Path p) {
97+
/* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
98+
* use getCanonicalPath in File to get the target of the symlink but that
99+
* does not indicate if the given path refers to a symlink.
100+
*/
101+
try {
102+
final String path = p.toUri().getPath();
103+
return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
104+
} catch (IOException x) {
105+
return "";
106+
}
107+
}
108+
109+
/**
110+
* Return a FileStatus representing the given path. If the path refers
111+
* to a symlink return a FileStatus representing the link rather than
112+
* the object the link refers to.
113+
*/
114+
@Override
115+
public FileStatus getFileLinkStatus(final Path f) throws IOException {
116+
String target = readLink(f);
117+
try {
118+
FileStatus fs = getFileStatus(f);
119+
// If f refers to a regular file or directory
120+
if ("".equals(target)) {
121+
return fs;
122+
}
123+
// Otherwise f refers to a symlink
124+
return new FileStatus(fs.getLen(),
125+
false,
126+
fs.getReplication(),
127+
fs.getBlockSize(),
128+
fs.getModificationTime(),
129+
fs.getAccessTime(),
130+
fs.getPermission(),
131+
fs.getOwner(),
132+
fs.getGroup(),
133+
new Path(target),
134+
f);
135+
} catch (FileNotFoundException e) {
136+
/* The exists method in the File class returns false for dangling
137+
* links so we can get a FileNotFoundException for links that exist.
138+
* It's also possible that we raced with a delete of the link. Use
139+
* the readBasicFileAttributes method in java.nio.file.attributes
140+
* when available.
141+
*/
142+
if (!"".equals(target)) {
143+
return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(),
144+
"", "", new Path(target), f);
145+
}
146+
// f refers to a file or directory that does not exist
147+
throw e;
148+
}
149+
}
150+
151+
@Override
152+
public Path getLinkTarget(Path f) throws IOException {
153+
/* We should never get here. Valid local links are resolved transparently
154+
* by the underlying local file system and accessing a dangling link will
155+
* result in an IOException, not an UnresolvedLinkException, so FileContext
156+
* should never call this function.
157+
*/
158+
throw new AssertionError();
159+
}
160+
161+
}

0 commit comments

Comments
 (0)