diff --git a/.gitignore b/.gitignore index 8ebe388..530bf35 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ notebook *~ spark* zeppelin +oracle local-repo conf diff --git a/.travis.yml b/.travis.yml index 172b547..4bddc3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ env: - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=1.6.3 HADOOP_VERSION=2.6 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.0.2 HADOOP_VERSION=2.7 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 + - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.7 + - ZEPPELIN_VERSION=v0.8.1 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 before_script: - sudo apt-get update @@ -22,10 +24,10 @@ before_script: - npm install -g bower grunt script: - - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION + - travis_wait 30 ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - - export REPO=frosner/zeppelind + - export REPO=${DOCKER_NAMESPACE:-datascienceplatform}/zeppelind - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g') - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi` - docker build -t $REPO:$COMMIT . diff --git a/Dockerfile b/Dockerfile index 85c160b..5321a61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,14 +16,41 @@ RUN \ libglib2.0-0 \ libxext6 \ libsm6 \ - libxrender1 && \ + libxrender1 \ + libaio1 \ + build-essential \ + unzip && \ apt-get clean all +# INSTALL ORACLE INSTANT CLIENT +RUN mkdir -p opt/oracle +ADD ./oracle/linux/ . +RUN unzip instantclient-basic-linux.x64.zip -d /opt/oracle \ + && unzip instantclient-sdk-linux.x64.zip -d /opt/oracle \ + && unzip instantclient-sqlplus-linux.x64.zip -d /opt/oracle \ + && mv /opt/oracle/instantclient_* /opt/oracle/instantclient \ + && ln -s /opt/oracle/instantclient/libclntsh.so.* /opt/oracle/instantclient/libclntsh.so \ + && ln -s /opt/oracle/instantclient/libocci.so.* /opt/oracle/instantclient/libocci.so +RUN mkdir -p opt/oracle/instantclient/network/admin + +ENV OCI_HOME="/opt/oracle/instantclient" +ENV OCI_LIB_DIR="/opt/oracle/instantclient" +ENV OCI_INCLUDE_DIR="/opt/oracle/instantclient/sdk/include" +ENV OCI_VERSION=12 +ENV LD_LIBRARY_PATH="/opt/oracle/instantclient" +ENV TNS_ADMIN="/opt/oracle/instantclient" +ENV ORACLE_BASE="/opt/oracle/instantclient" +ENV ORACLE_HOME="/opt/oracle/instantclient" +ENV PATH="/opt/oracle/instantclient:$PATH" +RUN echo '/opt/oracle/instantclient/' | tee -a /etc/ld.so.conf.d/oracle_instant_client.conf && ldconfig + + RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ wget --quiet https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh -O ~/anaconda.sh && \ /bin/bash ~/anaconda.sh -b -p /opt/conda && \ rm ~/anaconda.sh +RUN /opt/conda/bin/conda install SQLAlchemy psycopg2 pymssql cx_Oracle RUN ln -s /opt/conda/bin/python /usr/bin/python ADD spark /usr/local/spark diff --git a/README.md b/README.md index 708d9e5..93d78b8 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Docker Zeppelin -[![Build Status](https://travis-ci.org/FRosner/docker-zeppelin.svg?branch=master)](https://travis-ci.org/FRosner/docker-zeppelin) -[![Docker Pulls](https://img.shields.io/docker/pulls/frosner/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/frosner/zeppelind/) -[![Image Size](https://images.microbadger.com/badges/image/frosner/zeppelind.svg)](https://microbadger.com/images/frosner/zeppelind "Get your own image badge on microbadger.com") +[![Build Status](https://travis-ci.org/Data-Science-Platform/docker-zeppelin.svg?branch=master)](https://travis-ci.org/Data-Science-Platform/docker-zeppelin) +[![Docker Pulls](https://img.shields.io/docker/pulls/datascienceplatform/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/datascienceplatform/zeppelind/) +[![Image Size](https://images.microbadger.com/badges/image/data-science-platform/zeppelind.svg)](https://microbadger.com/images/datascienceplatform/zeppelind "Get your own image badge on microbadger.com") ## Description @@ -10,7 +10,7 @@ Docker image for starting [Apache Zeppelin](https://zeppelin.apache.org/). ## Usage -You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/FRosner/nomad-docker-wrapper) if you are running your containers on Nomad. +You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/Data-Science-Platform/nomad-docker-wrapper) if you are running your containers on Nomad. ``` docker run -p 8080:8080 \ @@ -29,7 +29,7 @@ docker run -p 8080:8080 \ -v $(pwd)/notebooks:/usr/local/zeppelin/notebooks \ -v $(pwd)/conf:/usr/local/zeppelin/conf \ -v $(pwd)/hive:/hive \ - frosner/zeppelind:latest-zv0.6.2-s2.0.2-h2.7 + datascienceplatform/zeppelind:latest-zv0.6.2-s2.0.2-h2.7 ``` ## Configuration @@ -50,3 +50,67 @@ The docker image requires some environment variables to be set. They are used to | `ZEPPELIN_SPARK_DRIVER_MEMORY` | Amount of memory to allocate to the Spark driver process (e.g. `512M`). | | `ZEPPELIN_PYSPARK_PYTHON` | Path to python executable for the Spark worker nodes. | | `ZEPPELIN_MEM` | Zeppelin JVM Options | + +## Travis CI/CD + +These environment variables should be defined and set appropriately in your Travis CI Settings. +https://travis-ci.org//docker-zeppelin/settings + +| Variable | Description | +| -------- | ----------- | +| `DOCKER_USER` | Your docker ID | +| `DOCKER_PASSWORD` | Password for your docker ID (make sure "display value in build log" is disabled for this env var)| +| `DOCKER_NAMESPACE` | Docker namespace, where the image(s) should be pushed; usually equals to DOCKER_USER; if empty, then "datascienceplatform" (that can lead to permission issues while pushing) | + +## SQL Database Support + +SQL databases are supported trough [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) (a python package with a comprehensive set of tools for working with databases). This container includes the following [dialects](https://docs.sqlalchemy.org/en/latest/dialects/): + +| Dialect | Target DB | +| ------- | --------- | +| `pymssql` | Microsoft SQL Server | +| `psycopg2` | PostgreSQL | +| `cx_Oracle` | Oracle | + +Support for additional databases can be added by installing additional [dialects](https://docs.sqlalchemy.org/en/latest/dialects/) into an anaconda environment and setting `ZEPPELIN_PYSPARK_PYTHON` to the environment location. + +### Microsoft SQL Server Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "mssql+pymssql://:@:" +engine = create_engine(conn_str) +# List All DBs +res = engine.execute('SELECT * FROM master.sys.databases') +for row in res: + print row +``` + +### PostgreSQL Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "postgresql+psycopg2:///:@:" +engine = create_engine(conn_str) +# List All DBs +res = engine.execute('SELECT * FROM pg_database') +for row in res: + print row +``` + +### Oracle Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "oracle+cx_oracle:///:@:/" +engine = create_engine(conn_str) +res = engine.execute('SELECT * FROM my_table') +for row in res: + print row +``` diff --git a/build b/build index 1e21c62..84d5b3d 100755 --- a/build +++ b/build @@ -11,6 +11,12 @@ case "$spark_version" in "2.1"*) zeppelin_spark_version="2.1" ;; + "2.2"*) + zeppelin_spark_version="2.2" + ;; + "2.3"*) + zeppelin_spark_version="2.3" + ;; *) echo "Spark version '$spark_version' not supported." exit 1 @@ -30,15 +36,38 @@ zeppelin_hadoop_version="$hadoop_version" zeppelin_version="$3" +oracle_instantclient_version=${4:-12.2.0.1.0} + set -e -v +mkdir -p ./oracle/linux + +curl \ + -L -o oracle/linux/instantclient-sdk-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sdk-linux.x64-$oracle_instantclient_version.zip + +curl \ + -L -o oracle/linux/instantclient-sqlplus-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sqlplus-linux.x64-$oracle_instantclient_version.zip + +curl \ + -L -o oracle/linux/instantclient-basic-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-basic-linux.x64-$oracle_instantclient_version.zip + curl -s http://archive.apache.org/dist/spark/spark-$spark_version/spark-$spark_version-bin-hadoop$hadoop_version.tgz | tar -xz -C . mv spark-* spark git clone https://github.com/apache/zeppelin.git zeppelin cd zeppelin git checkout $zeppelin_version -mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' +if [ "$zeppelin_spark_version" == "2.3" ] +then + # https://github.com/apache/zeppelin/pull/2880 + # ./dev/change_scala_version.sh 2.11 + mvn -e clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 10 == 1' +else + mvn -e clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' +fi cd .. mv zeppelin/zeppelin-distribution/target/zeppelin-*.tar.gz zeppelin_dist.tar.gz rm -rf zeppelin diff --git a/start-zeppelin.sh b/start-zeppelin.sh index 6277dc5..e324d7d 100755 --- a/start-zeppelin.sh +++ b/start-zeppelin.sh @@ -30,7 +30,13 @@ replace_env_config log4j.properties replace_env_config hive-site.xml # add zeppelin group if not exists -if getent group $ZEPPELIN_PROCESS_GROUP_NAME; then +if [ -z "$ZEPPELIN_PROCESS_GROUP_NAME" ]; then + echo "Environment variable ZEPPELIN_PROCESS_GROUP_NAME required, but not set, exiting ..." + exit +elif [ -z "$ZEPPELIN_PROCESS_GROUP_ID" ]; then + echo "Environment variable ZEPPELIN_PROCESS_GROUP_ID required, but not set, exiting ..." + exit +elif getent group $ZEPPELIN_PROCESS_GROUP_NAME; then echo "Group $ZEPPELIN_PROCESS_GROUP_NAME already exists" else echo "Group $ZEPPELIN_PROCESS_GROUP_NAME does not exist, creating it with gid=$ZEPPELIN_PROCESS_GROUP_ID" @@ -38,7 +44,13 @@ else fi # add zeppelin user if not exists -if id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then +if [ -z "$ZEPPELIN_PROCESS_USER_NAME" ]; then + echo "Environment variable ZEPPELIN_PROCESS_USER_NAME required, but not set, exiting ..." + exit +elif [ -z "$ZEPPELIN_PROCESS_USER_ID" ]; then + echo "Environment variable ZEPPELIN_PROCESS_USER_ID required, but not set, exiting ..." + exit +elif id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then echo "User $ZEPPELIN_PROCESS_USER_NAME already exists" else echo "User $ZEPPELIN_PROCESS_USER_NAME does not exist, creating it with uid=$ZEPPELIN_PROCESS_USER_ID"