Merge branch 'master' into release-2.1

Author: michael-berlin

.gitignore

@@ -62,3 +62,6 @@ _test/
 
 # generated protobuf files
 /go/vt/.proto.tmp
+
+# Eclipse Java CheckStyle plugin configuration.
+/java/*/.checkstyle

.pullapprove.yml

@@ -27,4 +27,6 @@ groups:
       - thompsonja-bot
       - pivanof
       - pivanof-bot
+      - wangyipei01
+      - wangyipei01-bot
 

.travis.yml

@@ -2,7 +2,7 @@
 sudo: false
 language: go
 go:
-  - 1.7
+  - 1.8
 addons:
   apt:
     sources:
@@ -34,6 +34,7 @@ cache:
     # Cache downloaded and extracted MariaDB 10.0 packages.
     - $MYSQL_ROOT
     # Cache bootstrapped dependencies (e.g. protobuf and gRPC).
+    - $HOME/gopath/dist/etcd
     - $HOME/gopath/dist/grpc/.build_finished
     - $HOME/gopath/dist/grpc/usr/local
     - $HOME/gopath/dist/py-mock-1.0.1/.build_finished
@@ -44,16 +45,19 @@ cache:
     - $HOME/.m2
     - $HOME/.phpenv
 before_cache:
-  # Travis CI caching doesn't work with this symlink. Just delete it.
+  # Travis CI caching doesn't work with these symlinks. Just delete them.
+  - rm $HOME/gopath/bin/etcd
   - rm $HOME/gopath/bin/zksrv.sh
   # Delete these files because they keep changing (having the latest timestamp
   # of an update) and trigger a lengthy update of the cache (~25 seconds).
-  - rm $HOME/.m2/repository/com/youtube/vitess/*/maven-metadata-local.xml
-  - rm $HOME/.m2/repository/com/youtube/vitess/*/*-SNAPSHOT/*-SNAPSHOT.jar
-  - rm $HOME/.m2/repository/com/youtube/vitess/*/*-SNAPSHOT/_remote.repositories
-  - rm $HOME/.m2/repository/com/youtube/vitess/*/*-SNAPSHOT/maven-metadata-local.xml
-  - rm $HOME/.m2/repository/com/youtube/vitess/*/*-SNAPSHOT/resolver-status.properties
+  - rm $HOME/.m2/repository/io/vitess/*/maven-metadata-local.xml
+  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/*-SNAPSHOT*.jar
+  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/_remote.repositories
+  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/maven-metadata-local.xml
+  - rm $HOME/.m2/repository/io/vitess/*/*-SNAPSHOT/resolver-status.properties
+  - rm $HOME/.m2/repository/io/grpc/grpc-core/resolver-status.properties
   - rm $HOME/.m2/repository/io/grpc/protoc-gen-grpc-java/*/protoc-gen-grpc-java-*.pom.lastUpdated
+  - rm $HOME/.m2/repository/io/netty/netty-codec-http2/resolver-status.properties
   # Don't cache unnecessary PHP files.
   - rm $HOME/.phpenv/versions/*/sbin/*
   - rm $HOME/.phpenv/versions/*/bin/php-cgi
@@ -72,8 +76,13 @@ env:
     # As of 07/2015 this value works best in a Travis CI container.
     - VT_GO_PARALLEL_VALUE=4
     - PATH="$HOME/.phpenv/bin:$PATH"
-    # Add -follow to TEST_FLAGS below to print as the test runs, to diagnose stuck tests.
-    - TEST_FLAGS="-docker=false -timeout=5m -print-log -remote-stats=http://enisoc.com:15123/travis/stats"
+    # Note: The per test timeout must always be < 10 minutes because test.go
+    #       does not produce any log output while running and Travis kills a
+    #       build after 10 minutes without log output.
+    #       See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts
+    # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go
+    # will print the test's output.
+    - TEST_FLAGS="-docker=false -timeout=8m -print-log -remote-stats=http://enisoc.com:15123/travis/stats"
     - CC=gcc-4.8
     - CXX=g++-4.8
     # TODO: uncomment when php crashing is fixed
@@ -101,11 +110,13 @@ before_script:
   - source dev.env
   # Travis only tests. Run only on the first shard.
   # Part of "before_script" because in "script" the job would not fail immediately if a command fails.
-  # Webdriver tests are only on the last shard for non PRs. Start Sauce Connect only on this shard.
   - |
     if [[ $TEST_MATRIX = *"-shard 0"* ]]; then
       travis/check_make_proto.sh
-    elif [[ $TEST_MATRIX = *"-shard 4"* && $TRAVIS_PULL_REQUEST = "false" ]]; then
+    fi
+  # Webdriver tests are only on the last shard for non PRs. Start Sauce Connect only on this shard.
+  - |
+    if [[ $TEST_MATRIX = *"-shard 4"* && $TRAVIS_PULL_REQUEST = "false" ]]; then
       tools/sauce_connect_setup.sh
     fi
 script:

Makefile

@@ -30,13 +30,17 @@ endif
 build_web:
 	echo $$(date): Building web artifacts
 	cd web/vtctld2 && ng build -prod
+	cp -f web/vtctld2/src/{favicon.ico,plotly-latest.min.js,primeui-ng-all.min.css} web/vtctld2/dist/
 
 build:
 ifndef NOBANNER
 	echo $$(date): Building source tree
 endif
 	go install $(VT_GO_PARALLEL) -ldflags "$(tools/build_version_flags.sh)" ./go/...
 
+parser:
+	make -C go/vt/sqlparser
+
 # To pass extra flags, run test.go manually.
 # For example: go run test.go -docker=false -- --extra-flag
 # For more info see: go run test.go -help
@@ -69,11 +73,6 @@ unit_test_cover: build
 unit_test_race: build
 	tools/unit_test_race.sh
 
-# Run coverage and upload to coveralls.io.
-# Requires the secret COVERALLS_TOKEN env variable to be set.
-unit_test_goveralls: build
-	travis/goveralls.sh
-
 .ONESHELL:
 SHELL = /bin/bash
 
@@ -105,7 +104,7 @@ PROTOC_EXISTS := $(shell type -p $(PROTOC_DIR)/protoc)
 ifeq (,$(PROTOC_EXISTS))
   PROTOC_BINARY := $(shell which protoc)
   ifeq (,$(PROTOC_BINARY))
-    $(error "Cannot find protoc binary. Did you execute 'source dev.env'?")
+    $(error "Cannot find protoc binary. Did bootstrap.sh succeed, and did you execute 'source dev.env'?")
   endif
   PROTOC_DIR := $(dir $(PROTOC_BINARY))
 endif
@@ -175,7 +174,7 @@ docker_base:
 
 docker_base_mysql56:
 	chmod -R o=g *
-	docker build -f Dockerfile.percona -t vitess/base:mysql56 .
+	docker build -f Dockerfile.mysql56 -t vitess/base:mysql56 .
 
 docker_base_mariadb:
 	chmod -R o=g *

bootstrap.sh

@@ -45,7 +45,7 @@ if [ -f $zk_dist/.build_finished ]; then
 else
   rm -rf $zk_dist
   (cd $VTROOT/dist && \
-    wget http://archive.apache.org/dist/zookeeper/zookeeper-$zk_ver/zookeeper-$zk_ver.tar.gz && \
+    wget http://apache.org/dist/zookeeper/zookeeper-$zk_ver/zookeeper-$zk_ver.tar.gz && \
     tar -xzf zookeeper-$zk_ver.tar.gz && \
     mkdir -p $zk_dist/lib && \
     cp zookeeper-$zk_ver/contrib/fatjar/zookeeper-$zk_ver-fatjar.jar $zk_dist/lib && \
@@ -54,6 +54,42 @@ else
   touch $zk_dist/.build_finished
 fi
 
+# Download and install etcd, link etcd binary into our root.
+etcd_version=v3.1.0-rc.1
+etcd_dist=$VTROOT/dist/etcd
+etcd_version_file=$etcd_dist/version
+if [[ -f $etcd_version_file && "$(cat $etcd_version_file)" == "$etcd_version" ]]; then
+  echo "skipping etcd install. remove $etcd_version_file to force re-install."
+else
+  rm -rf $etcd_dist
+  mkdir -p $etcd_dist
+  download_url=https://github.com/coreos/etcd/releases/download
+  (cd $etcd_dist && \
+    wget ${download_url}/${etcd_version}/etcd-${etcd_version}-linux-amd64.tar.gz && \
+    tar xzf etcd-${etcd_version}-linux-amd64.tar.gz)
+  [ $? -eq 0 ] || fail "etcd download failed"
+  echo "$etcd_version" > $etcd_version_file
+fi
+ln -snf $etcd_dist/etcd-${etcd_version}-linux-amd64/etcd $VTROOT/bin/etcd
+
+# Download and install consul, link consul binary into our root.
+consul_version=0.7.2
+consul_dist=$VTROOT/dist/consul
+consul_version_file=$consul_dist/version
+if [[ -f $consul_version_file && "$(cat $consul_version_file)" == "$consul_version" ]]; then
+  echo "skipping consul install. remove $consul_version_file to force re-install."
+else
+  rm -rf $consul_dist
+  mkdir -p $consul_dist
+  download_url=https://releases.hashicorp.com/consul
+  (cd $consul_dist && \
+    wget ${download_url}/${consul_version}/consul_${consul_version}_linux_amd64.zip && \
+    unzip consul_${consul_version}_linux_amd64.zip)
+  [ $? -eq 0 ] || fail "consul download failed"
+  echo "$consul_version" > $consul_version_file
+fi
+ln -snf $consul_dist/consul $VTROOT/bin/consul
+
 # install gRPC C++ base, so we can install the python adapters.
 # this also installs protobufs
 grpc_dist=$VTROOT/dist/grpc
@@ -93,16 +129,17 @@ fi
 #
 # DO NOT ADD LIBRARY DEPENDENCIES HERE. Instead use govendor as described below.
 #
+# Note: We explicitly do not vendor the tools below because a) we want to stay
+# their latest version and b) it's easier to "go install" them this way.
 gotools=" \
        github.com/golang/lint/golint \
        github.com/golang/mock/mockgen \
        github.com/kardianos/govendor \
        golang.org/x/tools/cmd/goimports \
-       honnef.co/go/unused/cmd/unused \
+       golang.org/x/tools/cmd/goyacc \
+       honnef.co/go/tools/cmd/unused \
 "
 
-# Tools for uploading code coverage to coveralls.io (used by Travis CI).
-gotools+=" github.com/modocache/gover github.com/mattn/goveralls"
 # The cover tool needs to be installed into the Go toolchain, so it will fail
 # if Go is installed somewhere that requires root access.
 source tools/shell_functions.inc
@@ -135,6 +172,8 @@ ln -snf $VTTOP/data $VTROOT/data
 ln -snf $VTTOP/py $VTROOT/py-vtdb
 ln -snf $VTTOP/go/zk/zkctl/zksrv.sh $VTROOT/bin/zksrv.sh
 ln -snf $VTTOP/test/vthook-test.sh $VTROOT/vthook/test.sh
+ln -snf $VTTOP/test/vthook-test_backup_error $VTROOT/vthook/test_backup_error
+ln -snf $VTTOP/test/vthook-test_backup_transform $VTROOT/vthook/test_backup_transform
 
 # find mysql and prepare to use libmysqlclient
 if [ -z "$MYSQL_FLAVOR" ]; then
@@ -197,25 +236,29 @@ else
   cd $VTTOP/third_party/py && \
     tar -xzf mock-1.0.1.tar.gz && \
     cd mock-1.0.1 && \
-    python ./setup.py install --prefix=$mock_dist && \
+    $PYTHON ./setup.py install --prefix=$mock_dist && \
     touch $mock_dist/.build_finished && \
     cd .. && \
     rm -r mock-1.0.1
 fi
 
-# create pre-commit hooks
-echo "creating git pre-commit hooks"
+# Create the Git hooks.
+echo "creating git hooks"
 mkdir -p $VTTOP/.git/hooks
 ln -sf $VTTOP/misc/git/pre-commit $VTTOP/.git/hooks/pre-commit
+ln -sf $VTTOP/misc/git/prepare-commit-msg.bugnumber $VTTOP/.git/hooks/prepare-commit-msg
+ln -sf $VTTOP/misc/git/commit-msg.bugnumber $VTTOP/.git/hooks/commit-msg
+(cd $VTTOP && git config core.hooksPath $VTTOP/.git/hooks)
 
 # Download chromedriver
 echo "Installing selenium and chromedriver"
 selenium_dist=$VTROOT/dist/selenium
 mkdir -p $selenium_dist
-virtualenv $selenium_dist
-$selenium_dist/bin/pip install selenium
+$VIRTUALENV $selenium_dist
+PIP=$selenium_dist/bin/pip
+$PIP install selenium
 mkdir -p $VTROOT/dist/chromedriver
-curl -sL http://chromedriver.storage.googleapis.com/2.24/chromedriver_linux64.zip > chromedriver_linux64.zip
+curl -sL http://chromedriver.storage.googleapis.com/2.25/chromedriver_linux64.zip > chromedriver_linux64.zip
 unzip -o -q chromedriver_linux64.zip -d $VTROOT/dist/chromedriver
 rm chromedriver_linux64.zip
 

config/mycnf/rbr.cnf

@@ -0,0 +1 @@
+binlog-format=row

data/test/mysql_ldap_auth_config.json

@@ -0,0 +1,11 @@
+{
+		"LdapServer": "ldap.example.com:386",
+		"LdapCert": "path/to/ldap-client-cert.pem",
+		"LdapKey": "path/to/ldap-client-key.pem",
+		"LdapCA": "path/to/ldap-client-ca.pem",
+		"User": "uid=vitessROuser,ou=users,ou=people,dc=example,dc=com",
+		"Password": "sUpErSeCuRe1",
+		"GroupQuery": "ou=groups,ou=people,dc=example,dc=com",
+		"UserDnPattern": "uid=%s,ou=users,ou=people,dc=example,dc=com",
+		"RefreshSeconds": 300
+}

data/test/tabletserver/ddl_cases.txt

@@ -8,6 +8,11 @@
   "Action": "drop", "TableName": "b"
 }
 
+"drop table b.c"
+{
+  "Action": "drop", "TableName": "b.c"
+}
+
 "alter table c alter foo"
 {
   "Action": "alter", "TableName": "c", "NewTable": "c"
@@ -18,16 +23,31 @@
   "Action": "alter", "TableName": "c", "NewTable": "c"
 }
 
+"alter table b.c comment 'aa'"
+{
+  "Action": "alter", "TableName": "b.c", "NewTable": "b.c"
+}
+
 "drop index a on b"
 {
   "Action": "alter", "TableName": "b", "NewName": "b"
 }
 
+"drop index a on b.c"
+{
+  "Action": "alter", "TableName": "b.c", "NewName": "b.c"
+}
+
 "rename table a to b"
 {
   "Action": "rename", "TableName": "a", "NewTable": "b"
 }
 
+"rename table c.a to c.b"
+{
+  "Action": "rename", "TableName": "c.a", "NewTable": "c.b"
+}
+
 "alter table a rename b"
 {
   "Action": "rename", "TableName": "a", "NewTable": "b"
@@ -38,12 +58,17 @@
   "Action": "rename", "TableName": "a", "NewTable": "b"
 }
 
+"alter table c.a rename to c.b"
+{
+  "Action": "rename", "TableName": "c.a", "NewTable": "c.b"
+}
+
 "create view a asdasd"
 {
   "Action": "create", "NewName": "a"
 }
 
-"alter view c alter foo"
+"alter view c as foo"
 {
   "Action": "alter", "TableName": "c", "NewTable": "c"
 }