-
Notifications
You must be signed in to change notification settings - Fork 57
/
Dockerfile
44 lines (36 loc) · 1.29 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
FROM singularities/hadoop:2.8
MAINTAINER Singularities
# Version
ENV SPARK_VERSION=2.2.1
# Set home
ENV SPARK_HOME=/usr/local/spark-$SPARK_VERSION
# Install dependencies
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install \
-yq --no-install-recommends \
python python3 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Install Spark
RUN mkdir -p "${SPARK_HOME}" \
&& export ARCHIVE=spark-$SPARK_VERSION-bin-without-hadoop.tgz \
&& export DOWNLOAD_PATH=apache/spark/spark-$SPARK_VERSION/$ARCHIVE \
&& curl -sSL https://mirrors.ocf.berkeley.edu/$DOWNLOAD_PATH | \
tar -xz -C $SPARK_HOME --strip-components 1 \
&& rm -rf $ARCHIVE
COPY spark-env.sh $SPARK_HOME/conf/spark-env.sh
ENV PATH=$PATH:$SPARK_HOME/bin
# Ports
EXPOSE 6066 7077 8080 8081
# Copy start script
COPY start-spark /opt/util/bin/start-spark
# Fix environment for other users
RUN echo "export SPARK_HOME=$SPARK_HOME" >> /etc/bash.bashrc \
&& echo 'export PATH=$PATH:$SPARK_HOME/bin'>> /etc/bash.bashrc
# Add deprecated commands
RUN echo '#!/usr/bin/env bash' > /usr/bin/master \
&& echo 'start-spark master' >> /usr/bin/master \
&& chmod +x /usr/bin/master \
&& echo '#!/usr/bin/env bash' > /usr/bin/worker \
&& echo 'start-spark worker $1' >> /usr/bin/worker \
&& chmod +x /usr/bin/worker