Skip to content

Commit b178bbe

Browse files
committed
Do not localize s3 files
(cherry picked from commit db105db)
1 parent f50480d commit b178bbe

File tree

1 file changed

+7
-2
lines changed
  • resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn

1 file changed

+7
-2
lines changed

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,12 @@ private[spark] class Client(
532532
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
533533
val linkname = targetDir.map(_ + "/").getOrElse("") +
534534
destName.orElse(Option(localURI.getFragment())).getOrElse(localPath.getName())
535-
val destPath = copyFileToRemote(destDir, localPath, replication, symlinkCache)
535+
var destPath = localPath
536+
if (!localPath.toUri.getScheme.startsWith("s3")) {
537+
destPath = copyFileToRemote(destDir, localPath, replication, symlinkCache)
538+
} else {
539+
logInfo(s"Adding binary from location: $destPath to the distributed cache")
540+
}
536541
val destFs = FileSystem.get(destPath.toUri(), hadoopConf)
537542
distCacheMgr.addResource(
538543
destFs, hadoopConf, destPath, localResources, resType, linkname, statCache,
@@ -709,7 +714,7 @@ private[spark] class Client(
709714
pySparkArchives.foreach { f =>
710715
val uri = Utils.resolveURI(f)
711716
if (uri.getScheme != Utils.LOCAL_SCHEME) {
712-
distribute(f)
717+
distribute(f, LocalResourceType.ARCHIVE)
713718
}
714719
}
715720

0 commit comments

Comments
 (0)