org.apache.hadoop.fs.FileUtil 클래스

static boolean copyMerge(FileSystem srcFS, Path srcDir, FileSystem dstFS, Path dstFile, boolean deleteSource, Configuration conf, String addString)

 LOCATION 

 'hdfs://hadoop/goolgleplus/log'

 tblproperties ("orc.compress"="NONE")

scala> val rdd1 = sc.parallelize(List("Spark","Scala"))

rdd1: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[50] at parallelize at <console>:24

scala> val rdd2 = sc.parallelize(List("Akka","Scala"))

rdd2: org.apache.spark.rdd.RDD[String] = ParallelCollectionRDD[51] at parallelize at <console>:24

scala> rdd1.union(rdd2).collect()

res11: Array[String] = Array(Spark, Scala, Akka, Scala)

scala> rdd1.intersection(rdd2).collect()

res12: Array[String] = Array(Scala)

scala> rdd1.cartesian(rdd2).collect()

res13: Array[(String, String)] = Array((Spark,Akka), (Spark,Scala), (Scala,Akka), (Scala,Scala))

scala> rdd1.subtract(rdd2).collect()

res14: Array[String] = Array(Spark)

scala> val hash1 = sc.parallelize( Seq(("1", "A"), ("2", "B"), ("3", "C"), ("1","D")))

hash1: org.apache.spark.rdd.RDD[(String, String)] = ParallelCollectionRDD[64] at parallelize at <console>:24

scala> val hash2 = sc.parallelize( Seq(("1", "W"), ("2", "X"), ("3", "Y"), ("2","Z")))

hash2: org.apache.spark.rdd.RDD[(String, String)] = ParallelCollectionRDD[65] at parallelize at <console>:24

scala> hash1.join(hash2).collect()

res15: Array[(String, (String, String))] = Array((1,(A,W)), (1,(D,W)), (2,(B,X)), (2,(B,Z)), (3,(C,Y)))

scala> hash1.cogroup(hash2).collect()

res16: Array[(String, (Iterable[String], Iterable[String]))] = Array((1,(CompactBuffer(A, D),CompactBuffer(W))), (2,(CompactBuffer(B),CompactBuffer(X, Z))), (3,(CompactBuffer(C),CompactBuffer(Y))))

not found: org.jboss.interceptor#jboss-interceptor-api;1.1

resolvers += "JBoss" at "https://repository.jboss.org/"

resolvers += "JBoss" at "https://repository.jboss.org/"

$ cat  xxx.txt

     Date date = Calendar.getInstance().getTime();

        // Display a date in day, month, year format

        DateFormat formatter = new SimpleDateFormat("dd/MM/yyyy");

        String today = formatter.format(date);

        System.out.println("Today : " + today);

        // Display date with day name in a short format

        formatter = new SimpleDateFormat("EEE, dd/MM/yyyy");

        today = formatter.format(date);

        System.out.println("Today : " + today);

        // Display date with a short day and month name

        formatter = new SimpleDateFormat("EEE, dd MMM yyyy");

        today = formatter.format(date);

        System.out.println("Today : " + today);

        // Formatting date with full day and month name and show time up to

        // milliseconds with AM/PM

        formatter = new SimpleDateFormat("EEEE, dd MMMM yyyy, hh:mm:ss.SSS a");

        today = formatter.format(date);

        System.out.println("Today : " + today);

scala> val codes = sc.textFile("xxx.txt")

codes: org.apache.spark.rdd.RDD[String] = xxx.txt MapPartitionsRDD[1] at textFile at <console>:24

scala> val lower = codes.map( line =>line.toLowerCase)

lower: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[2] at map at <console>:26

scala> val words = lower.flatMap(line => line.split("\\s+"))

words: org.apache.spark.rdd.RDD[String] = MapPartitionsRDD[3] at flatMap at <console>:28

scala> val counts = words.map(word => (word, 1))

counts: org.apache.spark.rdd.RDD[(String, Int)] = MapPartitionsRDD[4] at map at <console>:30

scala> val frequency = counts.reduceByKey(_ + _)

frequency: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[5] at reduceByKey at <console>:32

scala> val invFrequency = frequency.map(_.swap)

invFrequency: org.apache.spark.rdd.RDD[(Int, String)] = MapPartitionsRDD[6] at map at <console>:34

scala> invFrequency.top(10).foreach(println)

(23,)

(9,=)

(6,date)

(5,//)

(4,with)

(4,today);)

(4,today)

(4,system.out.println("today)

(4,new)

(4,formatter.format(date);)

scala> val result = sc.textFile("xxx.txt").map( line =>line.toLowerCase).flatMap(line => line.split("\\s+")).map(word => (word, 1)).reduceByKey(_ + _).map(_.swap)

result: org.apache.spark.rdd.RDD[(Int, String)] = MapPartitionsRDD[21] at map at <console>:24

scala> result.top(10).foreach(println)

(23,)

(9,=)

(6,date)

(5,//)

(4,with)

(4,today);)

(4,today)

(4,system.out.println("today)

(4,new)

(4,formatter.format(date);)

scala> val stopWords = Set("", "=", "//", ")", "(", ";", ":", "+", "-", "\"")

stopWords: scala.collection.immutable.Set[String] = Set("", =, ), ", -, ;, //, +, (, :)

scala> val result = sc.textFile("xxx.txt").map( line =>line.toLowerCase).flatMap(line => line.split("\\s+")).filter(! stopWords.contains(_)).map(word => (word, 1)).reduceByKey(_ + _).map(_.swap)

result: org.apache.spark.rdd.RDD[(Int, String)] = MapPartitionsRDD[34] at map at <console>:26

scala> result.top(10).foreach(println)

(6,date)

(4,with)

(4,today);)

(4,today)

(4,system.out.println("today)

(4,new)

(4,formatter.format(date);)

(4,formatter)

(3,name)

(3,display)

java.util.NoSuchElementException: None.get 

17/03/14 20:33:06 ERROR Executor: Exception in task 1.0 in stage 4.0 (TID 33)

java.util.NoSuchElementException: None.get

at scala.None$.get(Option.scala:347)

at scala.None$.get(Option.scala:345)

at org.apache.spark.storage.BlockInfoManager.releaseAllLocksForTask(BlockInfoManager.scala:343)

at org.apache.spark.storage.BlockManager.releaseAllLocksForTask(BlockManager.scala:670)

at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:289)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

org.apache.spark.SparkException: Job aborted due to stage failure: Task 7 in stage 4.0 failed 1 times, most recent failure: Lost task 7.0 in stage 4.0 (TID 39, localhost, executor driver): java.util.NoSuchElementException: None.get

at scala.None$.get(Option.scala:347)

at scala.None$.get(Option.scala:345)

at org.apache.spark.storage.BlockInfoManager.releaseAllLocksForTask(BlockInfoManager.scala:343)

at org.apache.spark.storage.BlockManager.releaseAllLocksForTask(BlockManager.scala:670)

at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:289)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:

  at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)

  at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)

  at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)

  at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)

  at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)

  at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)

  at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)

  at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)

  at scala.Option.foreach(Option.scala:257)

  at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)

  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)

  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)

  at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)

  at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)

  at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)

  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)

  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)

  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1944)

  at org.apache.spark.SparkContext.runJob(SparkContext.scala:1958)

  at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:917)

  at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:915)

  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)

  at org.apache.spark.rdd.RDD.withScope(RDD.scala:362)

  at org.apache.spark.rdd.RDD.foreach(RDD.scala:915)

  ... 52 elided

Caused by: java.util.NoSuchElementException: None.get

  at scala.None$.get(Option.scala:347)

  at scala.None$.get(Option.scala:345)

  at org.apache.spark.storage.BlockInfoManager.releaseAllLocksForTask(BlockInfoManager.scala:343)

  at org.apache.spark.storage.BlockManager.releaseAllLocksForTask(BlockManager.scala:670)

  at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:289)

  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

  at java.lang.Thread.run(Thread.java:745)

Only one SparkContext may be running in this JVM (see SPARK-2243). To ignore this error, set spark.driver.allowMultipleContexts = true. The currently running SparkContext was created at:

scala> sc

res0: org.apache.spark.SparkContext = org.apache.spark.SparkContext@26a004ed

scala> val conf = new SparkConf().setAppName("aaa")

conf: org.apache.spark.SparkConf = org.apache.spark.SparkConf@64d42d3d

scala> val ssc = new StreamingContext(conf, Seconds(1))

org.apache.spark.SparkException: Only one SparkContext may be running in this JVM (see SPARK-2243). To ignore this error, set spark.driver.allowMultipleContexts = true. The currently running SparkContext was created at:

org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:860)

scala> val ssc = new StreamingContext(sc, Seconds(1))

ssc: org.apache.spark.streaming.StreamingContext = org.apache.spark.streaming.StreamingContext@55ff64fd

The main method in the given main class must be static

  val mainMethod = mainClass.getMethod("main", new Array[String](0).getClass)

   if (!Modifier.isStatic(mainMethod.getModifiers)) {

      throw new IllegalStateException("The main method in the given main class must be static")

   }

http://docs.scala-lang.org/ko/tutorials/scala-for-java-programmers.html

1. object HelloWorld {
2. def main(args: Array[String]) {
3. println("Hello, world!")
4. }
5. }
6. 


똑똑한 독자들은 이미 눈치챘겠지만 위의 예제에서 main 함수는 static이 아니다. Scala에는 정적 멤버(함수든 필드든)라는 개념이 아얘 존재하지 않는다. 클래스의 일부로 정적 멤버를 정의하는 대신에 Scala 프로그래머들은 정적이기 원하는 멤버들을 싱글턴 객체안에 선언한다.

List("a", "b", "c").padTo(5, "-")

res0: List[String] = List(a, b, c, -, -)

scala> case class Log(tokens: List[String]) {

     | val z1 = tokens(0)

     | val z2 = tokens(1)

     | val z3 = tokens(2)

     | val z4 = tokens(3)

     | val z5 = tokens(4)

     | }

defined class Log

scala> val l = new Log(List("a", "b", "c").padTo(5, "-"))

l: Log = Log(List(a, b, c, -, -))

scala> l.z1

res4: String = a

scala> l.z2

res5: String = b

scala> l.z3

res6: String = c

scala> l.z4

res7: String = -

scala> l.z5

res8: String = -