From 245c4cd9d93f87ede9e3d504c50d2f5268caf518 Mon Sep 17 00:00:00 2001 From: Roman Pogribnyi Date: Tue, 22 May 2018 16:15:19 +0200 Subject: [PATCH 01/22] Update Readme.md (links to Travis, dockerhub etc) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 708d9e5..00b98f0 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Docker Zeppelin -[![Build Status](https://travis-ci.org/FRosner/docker-zeppelin.svg?branch=master)](https://travis-ci.org/FRosner/docker-zeppelin) -[![Docker Pulls](https://img.shields.io/docker/pulls/frosner/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/frosner/zeppelind/) -[![Image Size](https://images.microbadger.com/badges/image/frosner/zeppelind.svg)](https://microbadger.com/images/frosner/zeppelind "Get your own image badge on microbadger.com") +[![Build Status](https://travis-ci.org/Data-Science-Platform/docker-zeppelin.svg?branch=master)](https://travis-ci.org/Data-Science-Platform/docker-zeppelin) +[![Docker Pulls](https://img.shields.io/docker/pulls/datascienceplatform/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/datascienceplatform/zeppelind/) +[![Image Size](https://images.microbadger.com/badges/image/data-science-platform/zeppelind.svg)](https://microbadger.com/images/frosner/zeppelind "Get your own image badge on microbadger.com") ## Description From 76f514704943451dba13a55df053ba26baccc306 Mon Sep 17 00:00:00 2001 From: Roman Pogribnyi Date: Tue, 22 May 2018 16:16:56 +0200 Subject: [PATCH 02/22] Update dockerhub build repository(travis builds) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 172b547..e3f7f0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ script: - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - - export REPO=frosner/zeppelind + - export REPO=datascienceplatform/zeppelind - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g') - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi` - docker build -t $REPO:$COMMIT . From 0dce2baf39f953300b9dc6325b211c38b3c232d9 Mon Sep 17 00:00:00 2001 From: Aleksandr Sorokoumov Date: Thu, 14 Jun 2018 10:42:32 +0200 Subject: [PATCH 03/22] Fix a link to microbadger --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00b98f0..3847059 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/Data-Science-Platform/docker-zeppelin.svg?branch=master)](https://travis-ci.org/Data-Science-Platform/docker-zeppelin) [![Docker Pulls](https://img.shields.io/docker/pulls/datascienceplatform/zeppelind.svg?maxAge=2592000)](https://hub.docker.com/r/datascienceplatform/zeppelind/) -[![Image Size](https://images.microbadger.com/badges/image/data-science-platform/zeppelind.svg)](https://microbadger.com/images/frosner/zeppelind "Get your own image badge on microbadger.com") +[![Image Size](https://images.microbadger.com/badges/image/data-science-platform/zeppelind.svg)](https://microbadger.com/images/datascienceplatform/zeppelind "Get your own image badge on microbadger.com") ## Description From 7c33d62bdb2e8c28020f1e6da450b6353a6a41c0 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 12:34:21 +0100 Subject: [PATCH 04/22] DSP-44 zeppelin 0.8.0 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e3f7f0d..55f3a5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=1.6.3 HADOOP_VERSION=2.6 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.0.2 HADOOP_VERSION=2.7 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 + - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.8.5 before_script: - sudo apt-get update From 9e41bbb6d0a8e7e658ba912de00e89d4222249dc Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 16:38:57 +0100 Subject: [PATCH 05/22] DSP-44 allow pushing to own docker ID --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 55f3a5c..986a780 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ script: - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - - export REPO=datascienceplatform/zeppelind + - export REPO=${DOCKER_TARGET_ID:-datascienceplatform}/zeppelind - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g') - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi` - docker build -t $REPO:$COMMIT . From fdf633a10714818b74ab602eb9e70215c6b43dec Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 17:31:28 +0100 Subject: [PATCH 06/22] DSP-44 update readme and change docker env var name for namespace --- .travis.yml | 3 ++- README.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 986a780..41e39c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,8 @@ script: - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - - export REPO=${DOCKER_TARGET_ID:-datascienceplatform}/zeppelind + - export COMMIT=${TRAVIS_COMMIT::8} + - export REPO=${DOCKER_NAMESPACE:-datascienceplatform}/zeppelind - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g') - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi` - docker build -t $REPO:$COMMIT . diff --git a/README.md b/README.md index 3847059..e09c07e 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,14 @@ The docker image requires some environment variables to be set. They are used to | `ZEPPELIN_SPARK_DRIVER_MEMORY` | Amount of memory to allocate to the Spark driver process (e.g. `512M`). | | `ZEPPELIN_PYSPARK_PYTHON` | Path to python executable for the Spark worker nodes. | | `ZEPPELIN_MEM` | Zeppelin JVM Options | + +## Travis CI/CD + +These environment variables should be defined and set appropriately in your Travis CI Settings. +https://travis-ci.org//docker-zeppelin/settings + +| Variable | Description | +| -------- | ----------- | +| `DOCKER_USER` | Your docker ID | +| `DOCKER_PASSWORD` | Password for your docker ID (make sure "display value in build log" is disabled for this env var)| +| `DOCKER_NAMESPACE` | Docker namespace, where the image(s) should be pushed; usually equals to DOCKER_USER; if empty, then "datascienceplatform" (that can lead to permission issues while pushing) | From 8f51f311854edbca1edff5d7edc1f8f5c7a0d5db Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 17:49:06 +0100 Subject: [PATCH 07/22] DSP-44 spark 2.3.2 supports hadoop up to 2.7 only --- .travis.yml | 2 +- build | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 41e39c7..6ab3a65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=1.6.3 HADOOP_VERSION=2.6 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.0.2 HADOOP_VERSION=2.7 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 - - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.8.5 + - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.7 before_script: - sudo apt-get update diff --git a/build b/build index 1e21c62..c03f657 100755 --- a/build +++ b/build @@ -11,6 +11,9 @@ case "$spark_version" in "2.1"*) zeppelin_spark_version="2.1" ;; + "2.3"*) + zeppelin_spark_version="2.1" + ;; *) echo "Spark version '$spark_version' not supported." exit 1 From 0ec41479647d38188fd73f156e92e0ac08e4793b Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 17:56:25 +0100 Subject: [PATCH 08/22] DSP-44 minor fixes --- .travis.yml | 1 - build | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ab3a65..6a140fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,6 @@ script: - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - - export COMMIT=${TRAVIS_COMMIT::8} - export REPO=${DOCKER_NAMESPACE:-datascienceplatform}/zeppelind - export NICE_BRANCH_NAME=$(echo $TRAVIS_BRANCH | sed -e 's/\//_/g') - export TAG=`if [ "$TRAVIS_BRANCH" == "master" ]; then echo "latest"; else echo "$NICE_BRANCH_NAME" ; fi` diff --git a/build b/build index c03f657..c9a6395 100755 --- a/build +++ b/build @@ -12,7 +12,7 @@ case "$spark_version" in zeppelin_spark_version="2.1" ;; "2.3"*) - zeppelin_spark_version="2.1" + zeppelin_spark_version="2.3" ;; *) echo "Spark version '$spark_version' not supported." From 7a513782a469600cd58f82c7f902617ae73fdd5f Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Wed, 14 Nov 2018 17:58:11 +0100 Subject: [PATCH 09/22] DSP-44 support spark 2.2.x --- build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build b/build index c9a6395..c4e71bf 100755 --- a/build +++ b/build @@ -11,6 +11,9 @@ case "$spark_version" in "2.1"*) zeppelin_spark_version="2.1" ;; + "2.2"*) + zeppelin_spark_version="2.2" + ;; "2.3"*) zeppelin_spark_version="2.3" ;; From 03aa07cc54c3a7e252656b145bf9d889bf03d2c5 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Thu, 15 Nov 2018 14:42:31 +0100 Subject: [PATCH 10/22] DSP-44 simple mvn package command for zeppelin 0.8 --- build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build b/build index c4e71bf..7639d57 100755 --- a/build +++ b/build @@ -44,7 +44,13 @@ mv spark-* spark git clone https://github.com/apache/zeppelin.git zeppelin cd zeppelin git checkout $zeppelin_version -mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' +# https://github.com/apache/zeppelin/pull/2880 +if [ "$zeppelin_spark_version" == "2.3" ] +then + mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' +else + mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' +fi cd .. mv zeppelin/zeppelin-distribution/target/zeppelin-*.tar.gz zeppelin_dist.tar.gz rm -rf zeppelin From 7f5eb68eb082c110043a8a93dc13cc438553aaa3 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Thu, 15 Nov 2018 14:53:50 +0100 Subject: [PATCH 11/22] DSP-44 spark 2.3 removed support for scala 2.10 --- build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build b/build index 7639d57..7997f8c 100755 --- a/build +++ b/build @@ -44,9 +44,10 @@ mv spark-* spark git clone https://github.com/apache/zeppelin.git zeppelin cd zeppelin git checkout $zeppelin_version -# https://github.com/apache/zeppelin/pull/2880 if [ "$zeppelin_spark_version" == "2.3" ] then + # https://github.com/apache/zeppelin/pull/2880 + ./dev/change_scala_version.sh 2.11 mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' else mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' From 60d2521f8bee57817f0888c7a3857bb6a9cabf69 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Thu, 15 Nov 2018 15:54:14 +0100 Subject: [PATCH 12/22] DSP-44 no need to change scala version to 2.11 explicitly --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 7997f8c..8953a95 100755 --- a/build +++ b/build @@ -47,7 +47,7 @@ git checkout $zeppelin_version if [ "$zeppelin_spark_version" == "2.3" ] then # https://github.com/apache/zeppelin/pull/2880 - ./dev/change_scala_version.sh 2.11 + # ./dev/change_scala_version.sh 2.11 mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' else mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' From d7e960d67c170d41fb85e6f8e8568d608f38a8b1 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Thu, 15 Nov 2018 16:59:35 +0100 Subject: [PATCH 13/22] DSP-44 avoid cancellation of build due to timout (no output for 10min) --- build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build b/build index 8953a95..0482dd4 100755 --- a/build +++ b/build @@ -48,7 +48,7 @@ if [ "$zeppelin_spark_version" == "2.3" ] then # https://github.com/apache/zeppelin/pull/2880 # ./dev/change_scala_version.sh 2.11 - mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' + mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 10 == 1' else mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' fi From 18c76c56a502e2128a11328f5836351f2bb09e29 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Tue, 20 Nov 2018 11:15:37 +0100 Subject: [PATCH 14/22] DSP-44 replace references to frosner by DSP --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e09c07e..68f08d1 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Docker image for starting [Apache Zeppelin](https://zeppelin.apache.org/). ## Usage -You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/FRosner/nomad-docker-wrapper) if you are running your containers on Nomad. +You can either start the image directly with Docker, or use the [Nomad-Docker-Wrapper](https://github.com/Data-Science-Platform/nomad-docker-wrapper) if you are running your containers on Nomad. ``` docker run -p 8080:8080 \ @@ -29,7 +29,7 @@ docker run -p 8080:8080 \ -v $(pwd)/notebooks:/usr/local/zeppelin/notebooks \ -v $(pwd)/conf:/usr/local/zeppelin/conf \ -v $(pwd)/hive:/hive \ - frosner/zeppelind:latest-zv0.6.2-s2.0.2-h2.7 + datascienceplatform/zeppelind:latest-zv0.6.2-s2.0.2-h2.7 ``` ## Configuration From 2b0ce866164fac1f7b3f05aa708666719e7fd3ad Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Tue, 20 Nov 2018 11:16:19 +0100 Subject: [PATCH 15/22] DSP-44 make sure env variables for user and group are set --- start-zeppelin.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/start-zeppelin.sh b/start-zeppelin.sh index 6277dc5..3d80397 100755 --- a/start-zeppelin.sh +++ b/start-zeppelin.sh @@ -30,7 +30,10 @@ replace_env_config log4j.properties replace_env_config hive-site.xml # add zeppelin group if not exists -if getent group $ZEPPELIN_PROCESS_GROUP_NAME; then +if [ -z "$ZEPPELIN_PROCESS_GROUP_NAME" ]; then + echo "Environment variable ZEPPELIN_PROCESS_GROUP_NAME required, but not set, exiting ..." + exit +elif getent group $ZEPPELIN_PROCESS_GROUP_NAME; then echo "Group $ZEPPELIN_PROCESS_GROUP_NAME already exists" else echo "Group $ZEPPELIN_PROCESS_GROUP_NAME does not exist, creating it with gid=$ZEPPELIN_PROCESS_GROUP_ID" @@ -38,7 +41,10 @@ else fi # add zeppelin user if not exists -if id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then +if [ -z "$ZEPPELIN_PROCESS_USER_NAME" ]; then + echo "Environment variable $ZEPPELIN_PROCESS_USER_NAME required, but not set, exiting ..." + exit +elif id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then echo "User $ZEPPELIN_PROCESS_USER_NAME already exists" else echo "User $ZEPPELIN_PROCESS_USER_NAME does not exist, creating it with uid=$ZEPPELIN_PROCESS_USER_ID" From 78ba34ede0a881d7e6c20c4fd81d32cff7f4cd98 Mon Sep 17 00:00:00 2001 From: Slavo Ng Date: Tue, 20 Nov 2018 12:28:14 +0100 Subject: [PATCH 16/22] DSP-44 check also if ID env variables set for user and group --- start-zeppelin.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/start-zeppelin.sh b/start-zeppelin.sh index 3d80397..e324d7d 100755 --- a/start-zeppelin.sh +++ b/start-zeppelin.sh @@ -33,6 +33,9 @@ replace_env_config hive-site.xml if [ -z "$ZEPPELIN_PROCESS_GROUP_NAME" ]; then echo "Environment variable ZEPPELIN_PROCESS_GROUP_NAME required, but not set, exiting ..." exit +elif [ -z "$ZEPPELIN_PROCESS_GROUP_ID" ]; then + echo "Environment variable ZEPPELIN_PROCESS_GROUP_ID required, but not set, exiting ..." + exit elif getent group $ZEPPELIN_PROCESS_GROUP_NAME; then echo "Group $ZEPPELIN_PROCESS_GROUP_NAME already exists" else @@ -42,7 +45,10 @@ fi # add zeppelin user if not exists if [ -z "$ZEPPELIN_PROCESS_USER_NAME" ]; then - echo "Environment variable $ZEPPELIN_PROCESS_USER_NAME required, but not set, exiting ..." + echo "Environment variable ZEPPELIN_PROCESS_USER_NAME required, but not set, exiting ..." + exit +elif [ -z "$ZEPPELIN_PROCESS_USER_ID" ]; then + echo "Environment variable ZEPPELIN_PROCESS_USER_ID required, but not set, exiting ..." exit elif id -u $ZEPPELIN_PROCESS_USER_NAME 2>/dev/null; then echo "User $ZEPPELIN_PROCESS_USER_NAME already exists" From cc260fe065e03859ee1dc4f595d2ccd39802c321 Mon Sep 17 00:00:00 2001 From: Jose Villaveces Date: Wed, 6 Mar 2019 10:10:17 +0100 Subject: [PATCH 17/22] Installed SQL alchemy + dialects for postgres and mssql --- Dockerfile | 1 + README.md | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/Dockerfile b/Dockerfile index 85c160b..d5f45e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ /bin/bash ~/anaconda.sh -b -p /opt/conda && \ rm ~/anaconda.sh +RUN /opt/conda/bin/conda install SQLAlchemy psycopg2 pymssql RUN ln -s /opt/conda/bin/python /usr/bin/python ADD spark /usr/local/spark diff --git a/README.md b/README.md index 68f08d1..194370a 100644 --- a/README.md +++ b/README.md @@ -61,3 +61,42 @@ https://travis-ci.org//docker-zeppelin/settings | `DOCKER_USER` | Your docker ID | | `DOCKER_PASSWORD` | Password for your docker ID (make sure "display value in build log" is disabled for this env var)| | `DOCKER_NAMESPACE` | Docker namespace, where the image(s) should be pushed; usually equals to DOCKER_USER; if empty, then "datascienceplatform" (that can lead to permission issues while pushing) | + +## SQL Database Support + +SQL databases are supported trough [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) (a python package with a comprehensive set of tools for working with databases). This container includes the following [dialects](https://docs.sqlalchemy.org/en/latest/dialects/): + +| Dialect | Target DB | +| ------- | --------- | +| `pymssql` | Microsoft SQL Server | +| `psycopg2` | PostgreSQL | + +Support for additional databases can be added by installing additional [dialects](https://docs.sqlalchemy.org/en/latest/dialects/) into an anaconda environment and setting `ZEPPELIN_PYSPARK_PYTHON` to the environment location. + +### Microsoft SQL Server Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "mssql+pymssql://:@:" +engine = create_engine(conn_str) +# List All DBs +res = engine.execute('SELECT * FROM master.sys.databases') +for row in res: + print row +``` + +### PostgreSQL Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "postgresql+psycopg2:///:@:" +engine = create_engine(conn_str) +# List All DBs +res = engine.execute('SELECT * FROM pg_database') +for row in res: + print row +``` From 616799132472639bab3a179f820bf869f85979f1 Mon Sep 17 00:00:00 2001 From: Stanislav Khotinok Date: Wed, 13 Mar 2019 12:55:38 +0100 Subject: [PATCH 18/22] DSP-2157 Added build for zeppelin 0.8.1 with spark 2.1.1 and hadoop 2.7 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 6a140fa..64857e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.0.2 HADOOP_VERSION=2.7 - ZEPPELIN_VERSION=v0.7.2 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 - ZEPPELIN_VERSION=v0.8.0 SPARK_VERSION=2.3.2 HADOOP_VERSION=2.7 + - ZEPPELIN_VERSION=v0.8.1 SPARK_VERSION=2.1.1 HADOOP_VERSION=2.7 before_script: - sudo apt-get update From 1f330af1b81b116ba31bee6819903bf82fb6da12 Mon Sep 17 00:00:00 2001 From: Jose Villaveces Date: Wed, 20 Mar 2019 16:43:39 +0100 Subject: [PATCH 19/22] ignoring oracle folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8ebe388..530bf35 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ notebook *~ spark* zeppelin +oracle local-repo conf From a7b2070a2bff2dd4cc9d548cfbc1f5407805ba9e Mon Sep 17 00:00:00 2001 From: Jose Villaveces Date: Wed, 20 Mar 2019 16:44:04 +0100 Subject: [PATCH 20/22] Installing oracle instantclient --- Dockerfile | 30 ++++++++++++++++++++++++++++-- build | 20 ++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5f45e2..5321a61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,15 +16,41 @@ RUN \ libglib2.0-0 \ libxext6 \ libsm6 \ - libxrender1 && \ + libxrender1 \ + libaio1 \ + build-essential \ + unzip && \ apt-get clean all +# INSTALL ORACLE INSTANT CLIENT +RUN mkdir -p opt/oracle +ADD ./oracle/linux/ . +RUN unzip instantclient-basic-linux.x64.zip -d /opt/oracle \ + && unzip instantclient-sdk-linux.x64.zip -d /opt/oracle \ + && unzip instantclient-sqlplus-linux.x64.zip -d /opt/oracle \ + && mv /opt/oracle/instantclient_* /opt/oracle/instantclient \ + && ln -s /opt/oracle/instantclient/libclntsh.so.* /opt/oracle/instantclient/libclntsh.so \ + && ln -s /opt/oracle/instantclient/libocci.so.* /opt/oracle/instantclient/libocci.so +RUN mkdir -p opt/oracle/instantclient/network/admin + +ENV OCI_HOME="/opt/oracle/instantclient" +ENV OCI_LIB_DIR="/opt/oracle/instantclient" +ENV OCI_INCLUDE_DIR="/opt/oracle/instantclient/sdk/include" +ENV OCI_VERSION=12 +ENV LD_LIBRARY_PATH="/opt/oracle/instantclient" +ENV TNS_ADMIN="/opt/oracle/instantclient" +ENV ORACLE_BASE="/opt/oracle/instantclient" +ENV ORACLE_HOME="/opt/oracle/instantclient" +ENV PATH="/opt/oracle/instantclient:$PATH" +RUN echo '/opt/oracle/instantclient/' | tee -a /etc/ld.so.conf.d/oracle_instant_client.conf && ldconfig + + RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ wget --quiet https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh -O ~/anaconda.sh && \ /bin/bash ~/anaconda.sh -b -p /opt/conda && \ rm ~/anaconda.sh -RUN /opt/conda/bin/conda install SQLAlchemy psycopg2 pymssql +RUN /opt/conda/bin/conda install SQLAlchemy psycopg2 pymssql cx_Oracle RUN ln -s /opt/conda/bin/python /usr/bin/python ADD spark /usr/local/spark diff --git a/build b/build index 0482dd4..84d5b3d 100755 --- a/build +++ b/build @@ -36,8 +36,24 @@ zeppelin_hadoop_version="$hadoop_version" zeppelin_version="$3" +oracle_instantclient_version=${4:-12.2.0.1.0} + set -e -v +mkdir -p ./oracle/linux + +curl \ + -L -o oracle/linux/instantclient-sdk-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sdk-linux.x64-$oracle_instantclient_version.zip + +curl \ + -L -o oracle/linux/instantclient-sqlplus-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-sqlplus-linux.x64-$oracle_instantclient_version.zip + +curl \ + -L -o oracle/linux/instantclient-basic-linux.x64.zip \ + -s https://github.com/bumpx/oracle-instantclient/raw/master/instantclient-basic-linux.x64-$oracle_instantclient_version.zip + curl -s http://archive.apache.org/dist/spark/spark-$spark_version/spark-$spark_version-bin-hadoop$hadoop_version.tgz | tar -xz -C . mv spark-* spark @@ -48,9 +64,9 @@ if [ "$zeppelin_spark_version" == "2.3" ] then # https://github.com/apache/zeppelin/pull/2880 # ./dev/change_scala_version.sh 2.11 - mvn clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 10 == 1' + mvn -e clean package -DskipTests -Pbuild-distr | perl -ne 'print if $. % 10 == 1' else - mvn clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' + mvn -e clean package -Pspark-$zeppelin_spark_version -Phadoop-$zeppelin_hadoop_version -DskipTests -Pbuild-distr | perl -ne 'print if $. % 100 == 1' fi cd .. mv zeppelin/zeppelin-distribution/target/zeppelin-*.tar.gz zeppelin_dist.tar.gz From e42d80fb445fb75f838354ae7be9a7f417f856fd Mon Sep 17 00:00:00 2001 From: Jose Villaveces Date: Wed, 20 Mar 2019 16:50:21 +0100 Subject: [PATCH 21/22] Documenting Oracle DB usage --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 194370a..93d78b8 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ SQL databases are supported trough [SQLAlchemy](https://docs.sqlalchemy.org/en/l | ------- | --------- | | `pymssql` | Microsoft SQL Server | | `psycopg2` | PostgreSQL | +| `cx_Oracle` | Oracle | Support for additional databases can be added by installing additional [dialects](https://docs.sqlalchemy.org/en/latest/dialects/) into an anaconda environment and setting `ZEPPELIN_PYSPARK_PYTHON` to the environment location. @@ -100,3 +101,16 @@ res = engine.execute('SELECT * FROM pg_database') for row in res: print row ``` + +### Oracle Example + +``` +%spark.pyspark +import sqlalchemy +from sqlalchemy import create_engine +conn_str = "oracle+cx_oracle:///:@:/" +engine = create_engine(conn_str) +res = engine.execute('SELECT * FROM my_table') +for row in res: + print row +``` From 28d1e8dc8c68c9d4102f12fe91715fdaba3c9250 Mon Sep 17 00:00:00 2001 From: Jose Villaveces Date: Wed, 27 Mar 2019 14:46:10 +0100 Subject: [PATCH 22/22] using travis wait so mvn will finish --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 64857e8..4bddc3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_script: - npm install -g bower grunt script: - - ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION + - travis_wait 30 ./build $SPARK_VERSION $HADOOP_VERSION $ZEPPELIN_VERSION - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - export COMMIT=${TRAVIS_COMMIT::8} - export REPO=${DOCKER_NAMESPACE:-datascienceplatform}/zeppelind