Skip to content

Commit

Permalink
Merge branch 'master' into vchui
Browse files Browse the repository at this point in the history
  • Loading branch information
stuclem authored Jan 11, 2018
2 parents c6514f4 + 4585d53 commit 0a174fc
Show file tree
Hide file tree
Showing 169 changed files with 1,765 additions and 7,809 deletions.
12 changes: 8 additions & 4 deletions .drone.local.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,33 +7,37 @@ workspace:

pipeline:
integration-test:
image: ${TEST_BUILD_IMAGE=gcr.io/eminent-nation-87317/vic-integration-test:1.36}
image: ${TEST_BUILD_IMAGE=gcr.io/eminent-nation-87317/vic-integration-test:1.37}
pull: true
privileged: true
environment:
BIN: bin
GOPATH: /go
PUBLIC_NETWORK: ${PUBLIC_NETWORK}
SHELL: /bin/bash
TEST_DATASTORE: ${TEST_DATASTORE}
TEST_PASSWORD: ${TEST_PASSWORD}
TEST_RESOURCE: ${TEST_RESOURCE}
TEST_RESOURCE: ${TEST_RESOURCE}
TEST_URL: ${TEST_URL}
TEST_USERNAME: ${TEST_USERNAME}
volumes:
- /tmp
commands:
- tests/robot-run.sh "-s 3-01-Admiral-UI"
- tests/robot-run.sh "-s 4-01-Harbor"

services:
selenium-grid-hub:
image: selenium/hub:3.7.1
ports:
- "4444:4444"
environment:
- GRID_TIMEOUT=180000
- GRID_BROWSER_TIMEOUT=180000

selenium-node-firefox:
image: selenium/node-firefox:3.7.1
environment:
SE_OPTS: "-port 4445"
HUB_PORT_4444_TCP_ADDR: 127.0.0.1
HUB_PORT_4444_TCP_PORT: 4444
DISPLAY: ":98.0"
DISPLAY: ":98.0"
114 changes: 107 additions & 7 deletions .drone.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,34 @@ pipeline:
branch: [ master, releases/*, refs/tags/* ]
status: success

integration-test-ova-setup:
image: harbor.ci.drone.local/library/vic-integration-test:1.37
pull: true
privileged: true
environment:
BIN: bin
GOPATH: /go
PUBLIC_NETWORK: ${PUBLIC_NETWORK}
SHELL: /bin/bash
TEST_DATASTORE: ${TEST_DATASTORE}
TEST_PASSWORD: ${TEST_PASSWORD}
TEST_RESOURCE: ${TEST_RESOURCE}
TEST_URL: ${TEST_URL}
TEST_USERNAME: ${TEST_USERNAME}
volumes:
- /tmp
commands:
- pybot -d robot-logs/ova-setup-logs -s OVA-Setup tests/common-ova
when:
repo: vmware/vic-product
event: [ push, pull_request, tag, deployment ]
branch: [ master, releases/*, refs/tags/* ]
status: success

integration-test:
image: harbor.ci.drone.local/library/vic-integration-test:1.36
image: harbor.ci.drone.local/library/vic-integration-test:1.37
pull: true
privileged: true
environment:
BIN: bin
GOPATH: /go
Expand All @@ -96,13 +121,70 @@ pipeline:
branch: [ master, releases/*, refs/tags/* ]
status: success

integration-test-ova-cleanup:
image: harbor.ci.drone.local/library/vic-integration-test:1.37
pull: true
environment:
BIN: bin
GOPATH: /go
PUBLIC_NETWORK: ${PUBLIC_NETWORK}
SHELL: /bin/bash
TEST_DATASTORE: ${TEST_DATASTORE}
TEST_PASSWORD: ${TEST_PASSWORD}
TEST_RESOURCE: ${TEST_RESOURCE}
TEST_URL: ${TEST_URL}
TEST_USERNAME: ${TEST_USERNAME}
volumes:
- /tmp
commands:
- pybot -d robot-logs/ova-cleanup-logs -s OVA-Cleanup tests/common-ova
when:
repo: vmware/vic-product
event: [ push, pull_request, tag, deployment ]
branch: [ master, releases/*, refs/tags/* ]
status: [ success, failure ]

bundle-robot-logs:
image: harbor.ci.drone.local/library/vic-integration-test:1.37
pull: true
environment:
SHELL: /bin/bash
commands:
- cp -R test-screenshots robot-logs 2>/dev/null
- mkdir robot-bundle
- cd robot-bundle
- outfile="ova_integration_logs_"$DRONE_BUILD_NUMBER"_"$DRONE_COMMIT".zip"
- zip -r $outfile ../robot-logs
- ls -l
- echo "Download test logs:"
- echo "https://console.cloud.google.com/m/cloudstorage/b/vic-ci-logs/o/$outfile?authuser=1"
when:
repo: vmware/vic-product
event: [ push, pull_request, tag, deployment ]
branch: [ master, releases/*, refs/tags/* ]
status: [ success, failure ]

publish-robot-logs:
image: victest/drone-gcs:1
pull: true
source: robot-bundle
target: vic-ci-logs
acl:
- allUsers:READER
cache_control: public,max-age=3600
when:
repo: vmware/vic-product
event: [ push, pull_request, tag, deployment ]
branch: [ master, releases/*, refs/tags/* ]
status: [ success, failure ]

bundle-dev-builds:
image: harbor.ci.drone.local/library/golang:1.8
pull: true
commands:
- mkdir bundle
- cd installer/bin
- TMP=$(echo "$(ls -1t | grep "\.ova")" | sed "s/-/-dev-${DRONE_BUILD_NUMBER}-/")
- TMP=$(echo "$(ls -1t | grep "\.ova")" | sed "s/-/-dev-/")
- echo "Passed build will have artifact at https://storage.googleapis.com/vic-product-ova-builds/$TMP"
- echo "Renaming build artifact to $TMP..."
- mv vic-*.ova ../../bundle/$TMP
Expand All @@ -119,7 +201,7 @@ pipeline:
commands:
- mkdir bundle
- cd installer/bin
- TMP=$(echo "$(ls -1t | grep "\.ova")" | sed "s/-/-stage-${DRONE_BUILD_NUMBER}-/")
- TMP=$(echo "$(ls -1t | grep "\.ova")" | sed "s/-/-stage-/")
- echo "Passed build will have artifact at https://storage.googleapis.com/vic-product-ova-builds/$TMP"
- echo "Renaming build artifact to $TMP..."
- mv vic-*.ova ../../bundle/$TMP
Expand All @@ -137,10 +219,9 @@ pipeline:
commands:
- mkdir bundle
- cd installer/bin
- TMP=$(echo "$(ls -1t | grep "\.ova")" | sed "s/-/-${DRONE_BUILD_NUMBER}-/")
- TMP=$(echo vic-*.ova)
- echo "Passed build will have artifact at https://storage.googleapis.com/vic-product-ova-releases/$TMP"
- echo "Renaming build artifact to $TMP..."
- mv vic-*.ova ../../bundle/$TMP
- mv vic-*.ova ../../bundle/
- ls -l ../../bundle
when:
repo: vmware/vic-product
Expand Down Expand Up @@ -238,11 +319,30 @@ services:
image: selenium/hub:3.7.1
ports:
- "4444:4444"
environment:
- GRID_TIMEOUT=180000
- GRID_BROWSER_TIMEOUT=180000

selenium-node-firefox:
selenium-node-firefox-1:
image: selenium/node-firefox:3.7.1
environment:
SE_OPTS: "-port 4445"
HUB_PORT_4444_TCP_ADDR: 127.0.0.1
HUB_PORT_4444_TCP_PORT: 4444
DISPLAY: ":97.0"

selenium-node-firefox-2:
image: selenium/node-firefox:3.7.1
environment:
SE_OPTS: "-port 4446"
HUB_PORT_4444_TCP_ADDR: 127.0.0.1
HUB_PORT_4444_TCP_PORT: 4444
DISPLAY: ":98.0"

selenium-node-firefox-3:
image: selenium/node-firefox:3.7.1
environment:
SE_OPTS: "-port 4447"
HUB_PORT_4444_TCP_ADDR: 127.0.0.1
HUB_PORT_4444_TCP_PORT: 4444
DISPLAY: ":99.0"
2 changes: 1 addition & 1 deletion .drone.yml.sig

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package.list

# robot
robot-logs/
pabot_results/
log.html
report.html
output.xml
Expand Down
5 changes: 3 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,16 +105,17 @@ curl https://cdn.rawgit.com/tommarshall/git-good-commit/v0.6.1/hook.sh > .git/ho

[dronevic]:https://ci.vcna.io/vmware/vic-product
[dronesrc]:https://github.com/drone/drone
[robotsrc]:https://github.com/robotframework/robotframework
[dronecli]:http://readme.drone.io/0.5/install/cli/
[commithook]:https://github.com/tommarshall/git-good-commit

## Automated Testing

Automated testing uses [Drone][dronesrc].
CI pipeline is setup using [Drone][dronesrc] and Automated integration testing using [Robot Framework][robotsrc].

PRs must pass unit tests and integration tests before being merged into `master`.

_TODO_ Document testing
For details, see [CI Workflow](installer/BUILD.md#ci-workflow) and [Automated Testing](tests/README.md)

## Reporting Bugs and Creating Issues

Expand Down
8 changes: 3 additions & 5 deletions docs/user_doc/SUMMARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* [Introduction](README.md)
* [Quick Start Links](vic_quickstart.md)
* [Overview](vic_overview/introduction.md)
* [Roles and Personas](vic_overview/roles_and_personas.md)

----

Expand Down Expand Up @@ -74,11 +75,8 @@
* [Command Line Certificate Login](vic_vsphere_admin/cmdline_login.md)
* [VCH Admin Status Reference](vic_vsphere_admin/vicadmin_status_ref.md)
* [Upgrading](vic_vsphere_admin/upgrading_vic.md)
* [Data Migration](vic_vsphere_admin/upgrade_data.md)
* [Pre-Upgrade Tasks](vic_vsphere_admin/pre_upgrade_tasks.md)
* [Upgrade the Appliance](vic_vsphere_admin/upgrade_appliance.md)
* [Post-Upgrade Tasks](vic_vsphere_admin/post_upgrade_tasks.md)
* [Troubleshoot Appliance Upgrade](vic_vsphere_admin/troubleshoot_appliance_upgrade.md)
* [Upgrade VCHs](vic_vsphere_admin/upgrade_vch.md)
* [VCH Upgrade Options](vic_vsphere_admin/upgrade_vch_options.md)
* [Upgrade Plug-In on Windows](vic_vsphere_admin/upgrade_h5_plugin_windows.md)
Expand All @@ -95,12 +93,12 @@
* [Container Volumes](vic_vsphere_admin/backup_volumes.md)
* [Backing Up VMDK Volumes](vic_vsphere_admin/backup_vmdk.md)
* [Restoring VMDK Volumes](vic_vsphere_admin/restore_vmdk.md)
* [Uninstall](vic_vsphere_admin/uninstall_vic.md)
* [Troubleshooting](vic_vsphere_admin/troubleshoot_vic.md)
* [Access Appliance Logs](vic_vsphere_admin/appliance_logs.md)
* [Access and Configure Appliance Logs](vic_vsphere_admin/appliance_logs.md)
* [Access VCH Logs](vic_vsphere_admin/log_bundles.md)
* [VCH Deployment Times Out](vic_vsphere_admin/ts_vch_deployment_timeout.md)
* [Appliance OVF Error](vic_vsphere_admin/ts_ovf_error.md)
* [Appliance Fails to Register with PSC](vic_vsphere_admin/ts_register_psc_fails.md)
* [Certificate Verification Error](vic_vsphere_admin/ts_thumbprint_error.md)
* [Browser Rejects Certificates](vic_vsphere_admin/ts_cert_error.md)
* [Missing Common Name Error Even When TLS Options Are Specified Correctly](vic_vsphere_admin/ts_cli_argument_error.md)
Expand Down
8 changes: 4 additions & 4 deletions docs/user_doc/pdf.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Download PDF Documentation #

- <a href="./pdf/vic_12_overview.pdf" target="_blank">Overview of vSphere Integrated Containers</a>
- <a href="./pdf/vic_12_vsphere_admin.pdf" target="_blank">Install, Deploy, and Maintain the vSphere Integrated Containers Infrastructure</a>
- <a href="./pdf/vic_12_cloud_admin.pdf" target="_blank">Configure and Manage vSphere Integrated Containers</a>
- <a href="./pdf/vic_12_app_dev.pdf" target="_blank">Develop Container Applications with vSphere Integrated Containers</a>
- <a href="./pdf/vic_13_overview.pdf" target="_blank">Overview of vSphere Integrated Containers</a>
- <a href="./pdf/vic_13_vsphere_admin.pdf" target="_blank">Install, Deploy, and Maintain the vSphere Integrated Containers Infrastructure</a>
- <a href="./pdf/vic_13_cloud_admin.pdf" target="_blank">Configure and Manage vSphere Integrated Containers</a>
- <a href="./pdf/vic_13_app_dev.pdf" target="_blank">Develop Container Applications with vSphere Integrated Containers</a>
4 changes: 2 additions & 2 deletions docs/user_doc/vic_app_dev/container_operations.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
|`ps`|[Show running containers](https://docs.docker.com/engine/reference/commandline/ps/)|Yes, since 1.0. Supports the `-a/--all`, `-f/--filter`, `--no-trunc`, and `-q/--quiet` options. Filtering by network name is supported, but filtering by network ID is not supported.|
|`rename`|[Rename a container](https://docs.docker.com/engine/reference/commandline/rename/)|Yes, since 1.1. Name resolution for renamed running containers is not supported, but if you restart the container the new name is resolved.|
|`restart`|[Restart a container](https://docs.docker.com/engine/reference/commandline/restart/)|Yes, since 1.0|
|`rm`|[Remove a container](https://docs.docker.com/engine/reference/commandline/rm/)|Yes, since 1.0. Removes associated anonymous and regular volumes. Supports the `--force` option and the `name` parameter. Does not support `docker rm -v`. To view volumes attached to a container that is removed, use `docker volume ls` and `docker volume inspect <id>`. If you continually invoke `docker create` to make more anonymous volumes, those volumes are left behind after each subsequent removal of that container.|
|`rm`|[Remove a container](https://docs.docker.com/engine/reference/commandline/rm/)|Yes, since 1.0. Supports the `--force` option and the `name` parameter. To view volumes attached to a container that is removed, use `docker volume ls` and `docker volume inspect <id>`. If you continually invoke `docker create` to make more anonymous volumes, those volumes are left behind after each subsequent removal of that container. <br>Since 1.3 `docker rm -v` is also supported. Running the command removes the container and any anonymous volumes joined to that container. If an anonymous volume is in use by another container, it is not removed. Named volumes (specified by name in the create/run command) are not deleted.|
|`run`|[Run a command in a new container](https://docs.docker.com/engine/reference/commandline/run/)| <a id="docker_run"></a>Yes, since 1.0. Supports mapping a random host port to the container when the host port is not specified. <br>Supports running images from private and custom registries.<br>`docker run --net=host` is not supported. You can specify a container network by using the [`--container-network`](../vic_vsphere_admin/container_networks.md) option when you deploy a virtual container host. Supports the `--attach`, `--cidfile`, `--cpuset-cpus`, `--detach`, `--detach-keys`, `--entrypoint`, `--env`, `--env-file`, `--help`, `--interactive`, `--ip`, `--link`, `--memory`, `--name`, `--net`, `--net-alias`, `--publish`, `--rm`, `--stop-signal`, `--stop-timeout`, `--tty`, `--user`, `--volume`, and `--workdir` options.|
|`start`|[Start a container](https://docs.docker.com/engine/reference/commandline/start/)|Yes, since 1.0. Supports the `--attach` and `--interactive` options.|
|`stats`|[Get container stats based on resource usage](https://docs.docker.com/engine/reference/commandline/stats/)|Yes. Provides statistics about CPU and memory usage since 1.1. Provides statistics about network or disk usage since 1.2.|
Expand Down Expand Up @@ -135,4 +135,4 @@ For information about Docker Compose file support, see [Supported Docker Compose

## Swarm Commands <a id="swarm"></a>

This version of vSphere Integrated Containers Engine does not directly support Docker Swarm. However, you can use the [`dch-photon` Docker Engine](build_push_images.md) to instantiate a Docker swarm for use with vSphere Integrated Containers.
This version of vSphere Integrated Containers Engine does not directly support Docker Swarm. However, you can use the [`dch-photon` Docker Engine](build_push_images.md) to instantiate a Docker swarm for use with vSphere Integrated Containers.
15 changes: 3 additions & 12 deletions docs/user_doc/vic_app_dev/deploy_multiple_docker_compose.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Having examined some of the considerations around deploying single containers to a Virtual Container Host (VCH), this section examples how to deploy applications that are comprised of multiple containers.

There are two approaches you can take to this. The most instictive approach would be to create scripts that manage the lifecycle of volumes, networks and containers.
There are two approaches you can take to this. The most instinctive approach would be to create scripts that manage the lifecycle of volumes, networks and containers.

The second approach is to use a manifest-based orchestrator such as Docker Compose. VIC 1.1 has some basic support for Docker Compose, but it is not functionally complete. Docker Compose is a proprietary orchestrator that drives the Docker API and ties other pieces of the Docker ecosystem together including Build and Swarm. Given that VIC engine doesn't currently support either Build or Swarm, Compose compatibility is necessarily limited. However, Compose can still be a useful tool, provided those limitations are understood.

Expand Down Expand Up @@ -48,7 +48,7 @@ docker network create --internal db-net
docker network create web-net
# start the database container - specify a subdirectory on the volume as the data dir
docker run -d --name $DB_CTR_NAME --net db-net -v db-data:/var/lib/mysql --cpus 1 -m 2g -e MYSQL_ROOT_PASSWORD=somewordpress -e MYSQL_DATABASE=$DB_PASSWORD -e MYSQL_USER=$DB_USER -e MYSQL_PASSWORD=wordpress mysql:5.7 --datadir=/var/lib/mysql/data
docker run -d --name $DB_CTR_NAME --net db-net -v db-data:/var/lib/mysql --cpus 1 -m 2g -e MYSQL_ROOT_PASSWORD=somewordpress -e MYSQL_DATABASE=$DB_PASSWORD -e MYSQL_USER=$DB_USER -e MYSQL_PASSWORD=wordpress mysql:5.7
# start the web container - note it resolves the database container by name over db-net
docker create --name $WEB_CTR_NAME --net web-net -p 8080:80 -v html-data:/var/www/html --cpus 2 -m 4g -e WORDPRESS_DB_HOST=$DB_CTR_NAME:3306 -e WORDPRESS_DB_USER=$DB_USER -e WORDPRESS_DB_PASSWORD=$DB_PASSWORD wordpress
Expand Down Expand Up @@ -200,7 +200,7 @@ docker-compose down --volumes --rmi # stop the application and remove all res

Given that VIC engine does not have a native build capability, it does not interpret the `build` keyword in a compose file and `docker-compose build` will not work when `DOCKER_HOST` points to a VIC endpoint. VIC engine relies upon the portability of the docker image format and it is expected that a regular docker engine will be used in a CI pipeline to build container images for test and deployment.

There are two ways to work around this. You can create separate Compose files for build and run, or you can use the same Compose file but just make sure to add a couple of arguments. We will explore both options here using another example of a Compose file that includes build instructions. In this case, the sample voting application found [here](https://github.com/dockersamples/example-voting-app/blob/master/docker-compose-simple.yml)
There are two ways to work around this. You can create separate Compose files for build and run, or you can use the same Compose file but just make sure to add a couple of arguments. We will explore both options here using another example of a Compose file that includes build instructions. In this case, the sample voting application found [here](https://github.com/dockersamples/example-voting-app/blob/master/docker-compose-simple.yml).

Let's start by cloning the repository: `git clone [email protected]:dockersamples/example-voting-app.git` and we'll start by looking at `docker-compose-simple.yml`.

Expand Down Expand Up @@ -253,8 +253,6 @@ services:
db:
image: postgres:9.4
environment:
PGDATA: /var/lib/postgresql/data/data # Added as a workaround to /lost+found in volume root
result:
image: <registry-address>/<project>/result:0.1 # Fully-qualified image name
Expand All @@ -276,10 +274,6 @@ Local volume mounts are useful for development and testing as they allow source

VIC engine cannot map volumes from a local filesystem into a container because VIC engine containers are strongly isolated and don't share a common filesystem. Despite this, it is still possible in VIC to add state to a container by pre-populating a volume with data and mounting it (TBD: link to "Pre-populate a Volume").

- Workaround to `/lost+found` folder

In VIC a Volume is an ext4 formatted disk. As such, it has `/lost+found` in the root. Some containers will not write data into this directory due to the presence of this folder, so in this case of postgres above, it is configured to create and write to a subdirectory of the mount point.

***Combining into a single Compose file***

If separate Compose files feels clunky, it's quite possible to build, push and run from the same Compose file. All we need to do is to merge them together and then make sure we tell docker-compose what we want. Here's an example of a merged file:
Expand All @@ -305,8 +299,6 @@ services:
db:
image: postgres:9.4
environment:
PGDATA: /var/lib/postgresql/data/data
result:
build: ./result
Expand Down Expand Up @@ -337,7 +329,6 @@ This is partly a question of functional completeness of VIC engine docker API su

- VIC engine supports version 2 of the Compose File format.
- VIC engine has no native build support.
- VIC volumes are disks and when mounted, have a `/lost+found` folder created by ext4. For some containers - databases in particular - you will need to configure them to use a subdirectory of the volume. See MySQL example above.
- VIC containers take time to boot and thus may exhibit timing related issues. Eg. You may need to set `COMPOSE_HTTP_TIMEOUT` to a higher value than the default.
- VIC containers have no notion of local read-write shared storage.

Expand Down
Loading

0 comments on commit 0a174fc

Please sign in to comment.