openSUSE Build Service

Overview Repositories Revisions Requests Users Attributes Meta

File build.sh of Package spark (Revision c358465c8002b7bea07274dcd5c15380)

Currently displaying revision c358465c8002b7bea07274dcd5c15380 , Show latest

#!/bin/bash
set -xe
PROJECT_PREFIX=`readlink -e .`

cd src/spark-1.6.3

export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
$PROJECT_PREFIX/kit/apache-maven-3.3.9/bin/mvn \
    -Dmaven.repo.local=$PROJECT_PREFIX/kit/m2 --settings $PROJECT_PREFIX/kit/m2/settings.xml --strict-checksums \
    -o -Phadoop-2.4 -DskipTests install

# The following script was copied from make-distribution.sh in the Spark source repository

SPARK_HOME="$(pwd)"
DISTDIR="$SPARK_HOME/dist"
SPARK_HIVE=1

# Make directories
rm -rf "$DISTDIR"
mkdir -p "$DISTDIR/lib"

# Copy jars
cp $PROJECT_PREFIX/kit/m2/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar "$DISTDIR/lib/"
cp $PROJECT_PREFIX/kit/m2/com/yammer/metrics/metrics-core/2.2.0/metrics-core-2.2.0.jar "$DISTDIR/lib/"
cp $PROJECT_PREFIX/kit/m2/org/apache/kafka/kafka_2.10/0.8.2.1/kafka_2.10-0.8.2.1.jar "$DISTDIR/lib/"

cp "$SPARK_HOME"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/"
cp "$SPARK_HOME"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/"
# This will fail if the -Pyarn profile is not provided
# In this case, silence the error and ignore the return code of this command
cp "$SPARK_HOME"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" &> /dev/null || :

# Copy example sources (needed for python and SQL)
mkdir -p "$DISTDIR/examples/src/main"
cp -r "$SPARK_HOME"/examples/src/main "$DISTDIR/examples/src/"

if [ "$SPARK_HIVE" == "1" ]; then
  cp "$SPARK_HOME"/lib_managed/jars/datanucleus*.jar "$DISTDIR/lib/"
fi

# Copy license and ASF files
cp "$SPARK_HOME/LICENSE" "$DISTDIR"
cp -r "$SPARK_HOME/licenses" "$DISTDIR"
cp "$SPARK_HOME/NOTICE" "$DISTDIR"

if [ -e "$SPARK_HOME"/CHANGES.txt ]; then
  cp "$SPARK_HOME/CHANGES.txt" "$DISTDIR"
fi

# Copy data files
cp -r "$SPARK_HOME/data" "$DISTDIR"

# Copy other things
mkdir "$DISTDIR"/conf
cp "$SPARK_HOME"/conf/*.template "$DISTDIR"/conf
cp "$SPARK_HOME/README.md" "$DISTDIR"
cp -r "$SPARK_HOME/bin" "$DISTDIR"
cp -r "$SPARK_HOME/python" "$DISTDIR"
cp -r "$SPARK_HOME/sbin" "$DISTDIR"
cp -r "$SPARK_HOME/ec2" "$DISTDIR"
# Copy SparkR if it exists
if [ -d "$SPARK_HOME"/R/lib/SparkR ]; then
  mkdir -p "$DISTDIR"/R/lib
  cp -r "$SPARK_HOME/R/lib/SparkR" "$DISTDIR"/R/lib
  cp "$SPARK_HOME/R/lib/sparkr.zip" "$DISTDIR"/R/lib
fi

# This file is required for command-line spark scripts to work
touch "$DISTDIR"/RELEASE

Places

File build.sh of Package spark (Revision c358465c8002b7bea07274dcd5c15380)

Places