From 13bcc0d2fd169a5fe9a2943159418c3da3181ebe Mon Sep 17 00:00:00 2001 From: Rupert Bailey Date: Sun, 3 Jun 2018 15:42:59 +1000 Subject: [PATCH 1/4] updated for spark 2.3.0 --- Dockerfile | 37 +++++++++++++++++++++++++------------ README.md | 18 +++++++++++++----- man1/run-spark.1 | 17 +++++++++++++++++ 3 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 man1/run-spark.1 diff --git a/Dockerfile b/Dockerfile index 5b07eca..bc0c374 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,19 +3,20 @@ FROM openjdk:8 MAINTAINER Prashanth Babu # Scala related variables. -ARG SCALA_VERSION=2.12.2 +ARG SCALA_VERSION=2.11.8 ARG SCALA_BINARY_ARCHIVE_NAME=scala-${SCALA_VERSION} ARG SCALA_BINARY_DOWNLOAD_URL=http://downloads.lightbend.com/scala/${SCALA_VERSION}/${SCALA_BINARY_ARCHIVE_NAME}.tgz # SBT related variables. -ARG SBT_VERSION=0.13.15 +ARG SBT_VERSION=1.1.2 ARG SBT_BINARY_ARCHIVE_NAME=sbt-$SBT_VERSION -ARG SBT_BINARY_DOWNLOAD_URL=https://dl.bintray.com/sbt/native-packages/sbt/${SBT_VERSION}/${SBT_BINARY_ARCHIVE_NAME}.tgz +ARG SBT_BINARY_DOWNLOAD_URL=https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/${SBT_BINARY_ARCHIVE_NAME}.tgz # Spark related variables. -ARG SPARK_VERSION=2.2.0 -ARG SPARK_BINARY_ARCHIVE_NAME=spark-${SPARK_VERSION}-bin-hadoop2.7 -ARG SPARK_BINARY_DOWNLOAD_URL=http://d3kbcqa49mib13.cloudfront.net/${SPARK_BINARY_ARCHIVE_NAME}.tgz +ARG SPARK_VERSION=2.3.0 +ARG SPARK_BINARY_ARCHIVE_PREFIX=spark-${SPARK_VERSION} +ARG SPARK_BINARY_ARCHIVE_NAME=${SPARK_BINARY_ARCHIVE_PREFIX}-bin-hadoop2.7 +ARG SPARK_BINARY_DOWNLOAD_URL=https://apache.org/dist/spark/${SPARK_BINARY_ARCHIVE_PREFIX}/${SPARK_BINARY_ARCHIVE_NAME}.tgz # Configure env variables for Scala, SBT and Spark. # Also configure PATH env variable to include binary folders of Java, Scala, SBT and Spark. @@ -25,20 +26,32 @@ ENV SPARK_HOME /usr/local/spark ENV PATH $JAVA_HOME/bin:$SCALA_HOME/bin:$SBT_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH # Download, uncompress and move all the required packages and libraries to their corresponding directories in /usr/local/ folder. -RUN apt-get -yqq update && \ - apt-get install -yqq vim screen tmux && \ +RUN echo 'deb http://security.debian.org/debian-security stretch/updates main' >>/etc/apt/sources.list && \ + apt-get -yqq update && \ + apt-get install -yqq vim screen tmux openssh-server && \ apt-get clean && \ + /etc/init.d/ssh start && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /tmp/* && \ wget -qO - ${SCALA_BINARY_DOWNLOAD_URL} | tar -xz -C /usr/local/ && \ - wget -qO - ${SBT_BINARY_DOWNLOAD_URL} | tar -xz -C /usr/local/ && \ wget -qO - ${SPARK_BINARY_DOWNLOAD_URL} | tar -xz -C /usr/local/ && \ + wget -qO - ${SBT_BINARY_DOWNLOAD_URL} | tar -xz -C /usr/local/ && \ cd /usr/local/ && \ ln -s ${SCALA_BINARY_ARCHIVE_NAME} scala && \ ln -s ${SPARK_BINARY_ARCHIVE_NAME} spark && \ cp spark/conf/log4j.properties.template spark/conf/log4j.properties && \ sed -i -e s/WARN/ERROR/g spark/conf/log4j.properties && \ - sed -i -e s/INFO/ERROR/g spark/conf/log4j.properties + sed -i -e s/INFO/ERROR/g spark/conf/log4j.properties && \ + echo 'scalaVersion := "2.11.8"' > /root/build.sbt + +RUN ssh-keygen -t RSA -f ~/.ssh/id_rsa -N '' && \ + mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && \ + ssh-keyscan localhost > ~/.ssh/known_hosts && \ + /etc/init.d/ssh start && \ + /usr/local/spark-2.3.0-bin-hadoop2.7/sbin/start-all.sh + + +#RUN apt-get install -yqq openssh-server # We will be running our Spark jobs as `root` user. USER root @@ -50,6 +63,6 @@ WORKDIR /root # SparkContext web UI on 4040 -- only available for the duration of the application. # Spark master’s web UI on 8080. # Spark worker web UI on 8081. -EXPOSE 4040 8080 8081 +EXPOSE 4040 8080 8081 22 -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/usr/local/spark/bin/spark-shell"] diff --git a/README.md b/README.md index 2500d88..3319ea5 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,25 @@ Apache Spark Docker image is available directly from [https://index.docker.io](h This image contains the following softwares: -* OpenJDK 64-Bit v1.8.0_131 -* Scala v2.12.2 -* SBT v0.13.15 +* OpenJDK 64-Bit v1.8.0_162 +* Scala v2.11.8 +* SBT v1.1.2 * Apache Spark v2.2.0 + ## Various versions of Spark Images Depending on the version of the Spark Image you want, please run the corresponding command.
Latest image is always the most recent version of Apache Spark available. As of 11th July, 2017 it is v2.2.0. -### Apache Spark latest [i.e. v2.2.0] +### Apache Spark latest [i.e. v2.3.0] +[Dockerfile for Apache Spark v2.3.0](https://github.com/A140233/docker-spark) + + docker pull A140233/my-spark:2.3.0 + new: run-spark shell script run run/rebuild a standalone container + see manpage: man -M . 1 run-spark + +### Apache Spark v2.2.0 [Dockerfile for Apache Spark v2.2.0](https://github.com/P7h/docker-spark) docker pull p7hb/docker-spark @@ -255,4 +263,4 @@ If you find any issues or would like to discuss further, please ping me on my Tw ## License [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright © 2016 Prashanth Babu.
-Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). \ No newline at end of file +Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). diff --git a/man1/run-spark.1 b/man1/run-spark.1 new file mode 100644 index 0000000..3a20467 --- /dev/null +++ b/man1/run-spark.1 @@ -0,0 +1,17 @@ +.\" Manpage for run-spark +.\" Contact rbailey working at AGL for errors +.TH run-spark 1 "03 Jun 2018" "1.0" "run-spark man page" +.SH NAME +run-spark \- run a spark container in local mode +.SH SYNOPSIS +\./run-spark [--restart|--reinit|--rebuild|-s|-i|-b] +.SH DESCRIPTION +run-spark is a helper script to build a spark instance. without arguments it will run the minimum to get the instance to work, other options will allow various points to restart the container, reinitialize/rerun the container or rebuild it from the dockerfile. This also uses different ports to standard to attempt to avoid conflict from an existing instance +.SH OPTIONS +The run-spark provides restart/reinit/rebuild options if your container is corrupted or needs changes +.SH SEE ALSO +docker(1) +.SH BUGS +No known bugs. +.SH AUTHOR +Rupert Bailey at AGL From 1253820a0a791a3ebd0e42beee12f49924af5ecc Mon Sep 17 00:00:00 2001 From: Rupert Bailey Date: Sun, 3 Jun 2018 15:45:21 +1000 Subject: [PATCH 2/4] removed comment --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc0c374..265c263 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,8 +51,6 @@ RUN ssh-keygen -t RSA -f ~/.ssh/id_rsa -N '' && \ /usr/local/spark-2.3.0-bin-hadoop2.7/sbin/start-all.sh -#RUN apt-get install -yqq openssh-server - # We will be running our Spark jobs as `root` user. USER root From d4522fe54baa0b7f02f44064b970cb8518ebcb66 Mon Sep 17 00:00:00 2001 From: Rupert Bailey Date: Sun, 3 Jun 2018 15:46:32 +1000 Subject: [PATCH 3/4] added container running script --- run-spark | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 run-spark diff --git a/run-spark b/run-spark new file mode 100755 index 0000000..3a9c21a --- /dev/null +++ b/run-spark @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +function usage(){ + echo "$0 [-s|-i|-b|--restart|--reinit|--rebuild]" + exit 1 +} + +TEMP=$(getopt -o sib --long restart,reinit,rebuild \ + -n "$0" -- "$@") + +if [ $? != 0 ] ; then usage >&2 ; exit 1 ; fi + +# Note the quotes around `$TEMP': they are essential! +eval set -- "$TEMP" + +mystart=false +init=false +build=false + +while true ; do + case "$1" in + -s | --restart ) mystart=true ; shift ;; + -i | --reinit ) init=true ; shift ;; + -b | --rebuild ) build=true ; shift ;; + -- ) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done +if [[ ${arg[0]} != '' ]]; then + echo "Remaining arguments:" + for arg do echo '--> '"\`$arg'" ; done +fi + +#stop the container if running and asked for restart +declare -a SPARK_CONTAINERS_RUNNING=$(docker ps --quiet --filter name=spark) +if [[ ${SPARK_CONTAINERS_RUNNING[0]} != '' && ( $mystart == true || $init == true || $build == true ) ]]; then docker stop $SPARK_CONTAINERS_RUNNING; fi + +#delete the container if existing and asked for reinit +declare -a SPARK_CONTAINERS_STOPPED=$(docker ps --quiet --all --filter name=spark) +if [[ ${SPARK_CONTAINERS_STOPPED[0]} != '' && ( $init == true || $build == true ) ]]; then docker rm $SPARK_CONTAINERS_STOPPED; fi + +#drop the image if visible and asked for rebuild +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +if [[ ${SPARK_IMAGES[0]} != '' && $build == true ]]; then docker rmi $SPARK_IMAGES; fi + +#build the new image if Dockerfile exists +if ! [[ -f 'Dockerfile' ]]; then echo "Dockerfile absent! Exiting..."; exit; fi + +#and only if image name is available +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +if [[ ${SPARK_IMAGES[0]} == '' ]]; then + echo "Docker is absent, building it..."; + docker build --tag my-spark:2.3.0 . +fi + +#run the container if image exists but container does not +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +declare -a SPARK_CONTAINERS=$(docker ps --quiet --all --filter name=spark) + +#exit if the image doesn't exist +if [[ (${SPARK_IMAGES[0]} == '' ) ]]; then echo "image doesn't exist! exiting..."; exit; fi + +#run if container does not exist +if [[ ( ${SPARK_CONTAINERS[0]} == '' ) ]]; then + echo "container doesn't exist! Running..." ; + docker run --detach --tty --publish 8022:22 --publish 4041:4040 --publish 8090:8080 --publish 8091:8081 --hostname spark --name=spark my-spark:2.3.0 +else + echo "container exists already..." +fi + +#start if start container isn't started +declare -a SPARK_CONTAINERS=$(docker ps --quiet --all --filter name=spark) +if [[ $(docker inspect -f {{.State.Running}} ${SPARK_CONTAINERS[0]}) == 'true' ]]; then + echo "container is running..." +else + echo "starting an existing container..." + docker start ${SPARK_CONTAINERS[0]} +fi From bfa6dcd7fbb1f368c922c18ff0de8e4184fec8a4 Mon Sep 17 00:00:00 2001 From: Rupert Bailey Date: Thu, 29 Aug 2019 17:00:39 +1000 Subject: [PATCH 4/4] working with scala 2.3.1 --- Dockerfile | 47 ++++++++++++++++++++++++++++++++++------------- run-spark | 20 +++++++++++--------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 265c263..1f26514 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,17 @@ FROM openjdk:8 MAINTAINER Prashanth Babu # Scala related variables. -ARG SCALA_VERSION=2.11.8 +ARG SCALA_VERSION=2.11.12 ARG SCALA_BINARY_ARCHIVE_NAME=scala-${SCALA_VERSION} ARG SCALA_BINARY_DOWNLOAD_URL=http://downloads.lightbend.com/scala/${SCALA_VERSION}/${SCALA_BINARY_ARCHIVE_NAME}.tgz # SBT related variables. -ARG SBT_VERSION=1.1.2 +ARG SBT_VERSION=1.2.3 ARG SBT_BINARY_ARCHIVE_NAME=sbt-$SBT_VERSION ARG SBT_BINARY_DOWNLOAD_URL=https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/${SBT_BINARY_ARCHIVE_NAME}.tgz # Spark related variables. -ARG SPARK_VERSION=2.3.0 +ARG SPARK_VERSION=2.3.1 ARG SPARK_BINARY_ARCHIVE_PREFIX=spark-${SPARK_VERSION} ARG SPARK_BINARY_ARCHIVE_NAME=${SPARK_BINARY_ARCHIVE_PREFIX}-bin-hadoop2.7 ARG SPARK_BINARY_DOWNLOAD_URL=https://apache.org/dist/spark/${SPARK_BINARY_ARCHIVE_PREFIX}/${SPARK_BINARY_ARCHIVE_NAME}.tgz @@ -26,11 +26,13 @@ ENV SPARK_HOME /usr/local/spark ENV PATH $JAVA_HOME/bin:$SCALA_HOME/bin:$SBT_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH # Download, uncompress and move all the required packages and libraries to their corresponding directories in /usr/local/ folder. -RUN echo 'deb http://security.debian.org/debian-security stretch/updates main' >>/etc/apt/sources.list && \ +# /etc/init.d/ssh start && \ +# apt-get install -yqq vim screen tmux openssh-server && \ +RUN echo 'deb http://security.debian.org/debian-security stretch/updates main' >>/etc/apt/sources.list && \ apt-get -yqq update && \ - apt-get install -yqq vim screen tmux openssh-server && \ + apt-get install -yqq apt-utils && \ + apt-get install -yqq vim screen tmux && \ apt-get clean && \ - /etc/init.d/ssh start && \ rm -rf /var/lib/apt/lists/* && \ rm -rf /tmp/* && \ wget -qO - ${SCALA_BINARY_DOWNLOAD_URL} | tar -xz -C /usr/local/ && \ @@ -42,18 +44,36 @@ RUN echo 'deb http://security.debian.org/debian-security stretch/updates main cp spark/conf/log4j.properties.template spark/conf/log4j.properties && \ sed -i -e s/WARN/ERROR/g spark/conf/log4j.properties && \ sed -i -e s/INFO/ERROR/g spark/conf/log4j.properties && \ - echo 'scalaVersion := "2.11.8"' > /root/build.sbt + cat<< EOF > /root/build.sbt + name := "my-spark" + version := "0.1.0" + scalaVersion := "${SCALA_VERSION}" + + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-streaming" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-sql" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-hive" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-streaming-twitter" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-mllib" % "${SPARK_VERSION}" withSources(), + "org.apache.spark" %% "spark-csv" % "${SPARK_VERSION}" withSources() + ) +EOF -RUN ssh-keygen -t RSA -f ~/.ssh/id_rsa -N '' && \ - mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && \ - ssh-keyscan localhost > ~/.ssh/known_hosts && \ - /etc/init.d/ssh start && \ - /usr/local/spark-2.3.0-bin-hadoop2.7/sbin/start-all.sh +#turned off sshd +#RUN ssh-keygen -t RSA -f ~/.ssh/id_rsa -N '' && \ +# mv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys && \ +# ssh-keyscan localhost > ~/.ssh/known_hosts && \ +# /etc/init.d/ssh start & && \ +RUN /usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7/sbin/start-all.sh # We will be running our Spark jobs as `root` user. USER root +# Allow jobs to be mounted externally +VOLUME ["/root/data","/root/scripts"] + # Working directory is set to the home folder of `root` user. WORKDIR /root @@ -61,6 +81,7 @@ WORKDIR /root # SparkContext web UI on 4040 -- only available for the duration of the application. # Spark master’s web UI on 8080. # Spark worker web UI on 8081. -EXPOSE 4040 8080 8081 22 +# EXPOSE 4040 8080 8081 22 #currently running with SSHD off +EXPOSE 4040 8080 8081 CMD ["/usr/local/spark/bin/spark-shell"] diff --git a/run-spark b/run-spark index 3a9c21a..d2d6d66 100755 --- a/run-spark +++ b/run-spark @@ -1,5 +1,7 @@ #!/usr/bin/env bash +SPARK_VERSION=`sed -n '/SPARK_VERSION/s/.*=\([0-9]\+\.[0-9]\+\.[0-9]\+\)/\1/p' Dockerfile` + function usage(){ echo "$0 [-s|-i|-b|--restart|--reinit|--rebuild]" exit 1 @@ -40,21 +42,21 @@ declare -a SPARK_CONTAINERS_STOPPED=$(docker ps --quiet --all --filter name=spar if [[ ${SPARK_CONTAINERS_STOPPED[0]} != '' && ( $init == true || $build == true ) ]]; then docker rm $SPARK_CONTAINERS_STOPPED; fi #drop the image if visible and asked for rebuild -declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:${SPARK_VERSION}) if [[ ${SPARK_IMAGES[0]} != '' && $build == true ]]; then docker rmi $SPARK_IMAGES; fi #build the new image if Dockerfile exists if ! [[ -f 'Dockerfile' ]]; then echo "Dockerfile absent! Exiting..."; exit; fi #and only if image name is available -declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:${SPARK_VERSION}) if [[ ${SPARK_IMAGES[0]} == '' ]]; then echo "Docker is absent, building it..."; - docker build --tag my-spark:2.3.0 . + docker build --tag my-spark:${SPARK_VERSION} . fi #run the container if image exists but container does not -declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:2.3.0) +declare -a SPARK_IMAGES=$(docker images --quiet --all my-spark:${SPARK_VERSION}) declare -a SPARK_CONTAINERS=$(docker ps --quiet --all --filter name=spark) #exit if the image doesn't exist @@ -62,17 +64,17 @@ if [[ (${SPARK_IMAGES[0]} == '' ) ]]; then echo "image doesn't exist! exiti #run if container does not exist if [[ ( ${SPARK_CONTAINERS[0]} == '' ) ]]; then - echo "container doesn't exist! Running..." ; - docker run --detach --tty --publish 8022:22 --publish 4041:4040 --publish 8090:8080 --publish 8091:8081 --hostname spark --name=spark my-spark:2.3.0 + echo "container doesn't exist, great, will spin it up..." ; + docker run --detach --tty --publish 8022:22 --publish 4041:4040 --publish 8090:8080 --publish 8091:8081 --hostname spark --name=spark my-spark:${SPARK_VERSION} else - echo "container exists already..." + echo "container exists already, will check if it needs starting instead..." fi #start if start container isn't started declare -a SPARK_CONTAINERS=$(docker ps --quiet --all --filter name=spark) if [[ $(docker inspect -f {{.State.Running}} ${SPARK_CONTAINERS[0]}) == 'true' ]]; then - echo "container is running..." + echo "...container is running" else - echo "starting an existing container..." + echo "...starting an existing container instead." docker start ${SPARK_CONTAINERS[0]} fi