DSP-1771: Add Oracle driver to Zeppelin notebook

.gitignore

@@ -3,5 +3,6 @@ notebook
 *~
 spark*
 zeppelin
+oracle
 local-repo
 conf

.travis.yml

@@ -14,6 +14,8 @@ env:
     - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=1.6.3 HADOOP_VERSION=2.6
     - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.0.2 HADOOP_VERSION=2.7
     - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7
+    - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.7
+    - ZEPPELIN_VERSION=v0.8.1 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7
 
 before_script:
   - sudo apt-get update
@@ -22,10 +24,10 @@ before_script:
   - npm install -g bower grunt
 
 script:
-  - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION
+  - travis_wait 30 ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION
   - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
   - export COMMIT=${TRAVIS_COMMIT::8}
-  - export REPO=frosner/zeppelind
+  - export REPO=${DOCKER_NAMESPACE:-datascienceplatform}/zeppelind
   - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g')
   - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi`
   - docker build -t $REPO:$COMMIT .

Dockerfile

@@ -16,14 +16,41 @@ RUN \
     libglib2.0-0 \
     libxext6 \
     libsm6 \
-    libxrender1 && \
+    libxrender1 \
+    libaio1 \
+    build-essential \
+    unzip && \
   apt-get clean all
 
+# INSTALL ORACLE INSTANT CLIENT
+RUN mkdir -p opt/oracle
+ADD ./oracle/linux/ .
+RUN unzip instantclient-basic-linux.x64.zip -d /opt/oracle \
+ && unzip instantclient-sdk-linux.x64.zip -d /opt/oracle  \
+ && unzip instantclient-sqlplus-linux.x64.zip -d /opt/oracle \
+ && mv /opt/oracle/instantclient_* /opt/oracle/instantclient \
+ && ln -s /opt/oracle/instantclient/libclntsh.so.* /opt/oracle/instantclient/libclntsh.so \
+ && ln -s /opt/oracle/instantclient/libocci.so.* /opt/oracle/instantclient/libocci.so
+RUN mkdir -p opt/oracle/instantclient/network/admin
+
+ENV OCI_HOME="/opt/oracle/instantclient"
+ENV OCI_LIB_DIR="/opt/oracle/instantclient"
+ENV OCI_INCLUDE_DIR="/opt/oracle/instantclient/sdk/include"
+ENV OCI_VERSION=12
+ENV LD_LIBRARY_PATH="/opt/oracle/instantclient"
+ENV TNS_ADMIN="/opt/oracle/instantclient"
+ENV ORACLE_BASE="/opt/oracle/instantclient"
+ENV ORACLE_HOME="/opt/oracle/instantclient"
+ENV PATH="/opt/oracle/instantclient:$PATH"
+RUN echo '/opt/oracle/instantclient/' | tee -a /etc/ld.so.conf.d/oracle_instant_client.conf && ldconfig
+
+
 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
   wget --quiet https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh -O ~/anaconda.sh && \
   /bin/bash ~/anaconda.sh -b -p /opt/conda && \
   rm ~/anaconda.sh
 
+RUN /opt/conda/bin/conda install SQLAlchemy psycopg2 pymssql cx_Oracle
 RUN ln -s /opt/conda/bin/python /usr/bin/python
 
 ADD spark /usr/local/spark

README.md

@@ -1,16 +1,16 @@
 # Docker Zeppelin
 
-[![Build Status](https://travis-ci.org/FRosner/docker-zeppelin.svg?branch=master)](https://travis-ci.org/FRosner/docker-zeppelin)
-[![Docker Pulls](https://img.shields.io/docker/pulls/frosner/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/frosner/zeppelind/)
-[![Image Size](https://images.microbadger.com/badges/image/frosner/zeppelind.svg)](https://microbadger.com/images/frosner/zeppelind "Get your own image badge on microbadger.com")
+[![Build Status](https://travis-ci.org/Data-Science-Platform/docker-zeppelin.svg?branch=master)](https://travis-ci.org/Data-Science-Platform/docker-zeppelin)
+[![Docker Pulls](https://img.shields.io/docker/pulls/datascienceplatform/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/datascienceplatform/zeppelind/)
+[![Image Size](https://images.microbadger.com/badges/image/data-science-platform/zeppelind.svg)](https://microbadger.com/images/datascienceplatform/zeppelind "Get your own image badge on microbadger.com")
 
 ## Description
 
 Docker image for starting [Apache Zeppelin](https://zeppelin.apache.org/).
 
 ## Usage
 
-You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/FRosner/nomad-docker-wrapper) if you are running your containers on Nomad.
+You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/Data-Science-Platform/nomad-docker-wrapper) if you are running your containers on Nomad.
 
 ```
 docker run -p 8080:8080 \
@@ -29,7 +29,7 @@ docker run -p 8080:8080 \
   -v $(pwd)/notebooks:/usr/local/zeppelin/notebooks \
   -v $(pwd)/conf:/usr/local/zeppelin/conf \
   -v $(pwd)/hive:/hive \
-  frosner/zeppelind:latest-zv0.6.2-s2.0.2-h2.7
+  datascienceplatform/zeppelind:latest-zv0.6.2-s2.0.2-h2.7
 ```
 
 ## Configuration
@@ -50,3 +50,67 @@ The docker image requires some environment variables to be set. They are used to
 | `ZEPPELIN_SPARK_DRIVER_MEMORY` | Amount of memory to allocate to the Spark driver process (e.g. `512M`). |
 | `ZEPPELIN_PYSPARK_PYTHON` | Path to python executable for the Spark worker nodes. |
 | `ZEPPELIN_MEM` | Zeppelin JVM Options |
+
+## Travis CI/CD
+
+These environment variables should be defined and set appropriately in your Travis CI Settings.
+https://travis-ci.org/<your travis account name>/docker-zeppelin/settings
+
+| Variable | Description |
+| -------- | ----------- |
+| `DOCKER_USER` | Your docker ID |
+| `DOCKER_PASSWORD` | Password for your docker ID (make sure "display value in build log" is disabled for this env var)|
+| `DOCKER_NAMESPACE` | Docker namespace, where the image(s) should be pushed; usually equals to DOCKER_USER; if empty, then "datascienceplatform" (that can lead to permission issues while pushing) |
+
+## SQL Database Support
+
+SQL databases are supported trough [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) (a python package  with a comprehensive set of tools for working with databases). This container includes the following [dialects](https://docs.sqlalchemy.org/en/latest/dialects/):
+
+| Dialect | Target DB |
+| ------- | --------- |
+| `pymssql` | Microsoft SQL Server |
+| `psycopg2` | PostgreSQL |
+| `cx_Oracle` | Oracle |
+
+Support for additional databases can be added by installing additional [dialects](https://docs.sqlalchemy.org/en/latest/dialects/) into an anaconda environment and setting `ZEPPELIN_PYSPARK_PYTHON` to the environment location.
+
+###  Microsoft SQL Server Example
+
+```
+%spark.pyspark
+import sqlalchemy
+from sqlalchemy import create_engine
+conn_str = "mssql+pymssql://<User>:<Password>@<Host>:<Port>"
+engine = create_engine(conn_str)
+# List All DBs
+res = engine.execute('SELECT * FROM master.sys.databases')
+for row in res:
+    print row
+```
+
+###  PostgreSQL Example
+
+```
+%spark.pyspark
+import sqlalchemy
+from sqlalchemy import create_engine
+conn_str = "postgresql+psycopg2:///<User>:<Password>@<Host>:<Port>"
+engine = create_engine(conn_str)
+# List All DBs
+res = engine.execute('SELECT * FROM pg_database')
+for row in res:
+    print row
+```
+
+###  Oracle Example
+
+```
+%spark.pyspark
+import sqlalchemy
+from sqlalchemy import create_engine
+conn_str = "oracle+cx_oracle:///<User>:<Password>@<Host>:<Port>/<db>"
+engine = create_engine(conn_str)
+res = engine.execute('SELECT * FROM my_table')
+for row in res:
+    print row
+```

build

@@ -11,6 +11,12 @@ case "$spark_version" in
   "2.1"*)
     zeppelin_spark_version="2.1"
     ;;
+  "2.2"*)
+    zeppelin_spark_version="2.2"
+    ;;
+  "2.3"*)
+    zeppelin_spark_version="2.3"
+    ;;
   *)
     echo "Spark version '$spark_version' not supported."
     exit 1
@@ -30,15 +36,38 @@ zeppelin_hadoop_version="$hadoop_version"
 
 zeppelin_version="$3"
 
+oracle_instantclient_version=${4:-12.2.0.1.0}
+
 set -e -v
 
+mkdir -p ./oracle/linux
+
+curl \
+    -L -o oracle/linux/instantclient-sdk-linux.x64.zip \
+    -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sdk-linux.x64-$oracle_instantclient_version.zip
+
+curl \
+    -L -o oracle/linux/instantclient-sqlplus-linux.x64.zip \
+    -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sqlplus-linux.x64-$oracle_instantclient_version.zip
+
+curl \
+    -L -o oracle/linux/instantclient-basic-linux.x64.zip \
+    -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-basic-linux.x64-$oracle_instantclient_version.zip
+
 curl -s http://archive.apache.org/dist/spark/spark-$spark_version/spark-$spark_version-bin-hadoop$hadoop_version.tgz | tar -xz -C .
 mv spark-* spark
 
 git clone https://github.com/apache/zeppelin.git zeppelin
 cd zeppelin
 git checkout $zeppelin_version
-mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1'
+if [ "$zeppelin_spark_version" == "2.3" ]
+then
+  # https://github.com/apache/zeppelin/pull/2880
+  # ./dev/change_scala_version.sh 2.11
+  mvn -e clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 10 == 1'
+else
+  mvn -e clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1'
+fi
 cd ..
 mv zeppelin/zeppelin-distribution/target/zeppelin-*.tar.gz zeppelin_dist.tar.gz
 rm -rf zeppelin

start-zeppelin.sh

@@ -30,15 +30,27 @@ replace_env_config log4j.properties
 replace_env_config hive-site.xml
 
 # add zeppelin group if not exists
-if getent group $ZEPPELIN_PROCESS_GROUP_NAME; then
+if [ -z "$ZEPPELIN_PROCESS_GROUP_NAME" ]; then
+  echo "Environment variable ZEPPELIN_PROCESS_GROUP_NAME required, but not set, exiting ..."
+  exit
+elif [ -z "$ZEPPELIN_PROCESS_GROUP_ID" ]; then
+  echo "Environment variable ZEPPELIN_PROCESS_GROUP_ID required, but not set, exiting ..."
+  exit
+elif getent group $ZEPPELIN_PROCESS_GROUP_NAME; then
   echo "Group $ZEPPELIN_PROCESS_GROUP_NAME already exists"
 else
   echo "Group $ZEPPELIN_PROCESS_GROUP_NAME does not exist, creating it with gid=$ZEPPELIN_PROCESS_GROUP_ID"
   addgroup --force-badname -gid $ZEPPELIN_PROCESS_GROUP_ID $ZEPPELIN_PROCESS_GROUP_NAME
 fi
 
 # add zeppelin user if not exists
-if id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then
+if [ -z "$ZEPPELIN_PROCESS_USER_NAME" ]; then
+  echo "Environment variable ZEPPELIN_PROCESS_USER_NAME required, but not set, exiting ..."
+  exit
+elif [ -z "$ZEPPELIN_PROCESS_USER_ID" ]; then
+  echo "Environment variable ZEPPELIN_PROCESS_USER_ID required, but not set, exiting ..."
+  exit
+elif id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then
   echo "User $ZEPPELIN_PROCESS_USER_NAME already exists"
 else
   echo "User $ZEPPELIN_PROCESS_USER_NAME does not exist, creating it with uid=$ZEPPELIN_PROCESS_USER_ID"