Merge pull request #1823 from dotcloud/1766-nightly-release

testing, issue #1766: Add nightly release to docker-ci
2026-08-03 22:51:03 +00:00 · 2013-10-11 17:38:06 -07:00
parent 6d6097cddf d542318d63
commit 620bc5b9e1
24 changed files with 495 additions and 204 deletions
--- a/hack/infrastructure/docker-ci.rst
+++ b/hack/infrastructure/docker-ci.rst
@@ -1,5 +1,38 @@
-docker-ci github pull request
-=============================
+docker-ci
+=========
+
+docker-ci is our buildbot continuous integration server,
+building and testing docker, hosted on EC2 and reachable at
+http://docker-ci.dotcloud.com
+
+
+Deployment
+==========
+
+# Load AWS credentials
+export AWS_ACCESS_KEY_ID=''
+export AWS_SECRET_ACCESS_KEY=''
+export AWS_KEYPAIR_NAME=''
+export AWS_SSH_PRIVKEY=''
+
+# Load buildbot credentials and config
+export BUILDBOT_PWD=''
+export IRC_PWD=''
+export IRC_CHANNEL='docker-dev'
+export SMTP_USER=''
+export SMTP_PWD=''
+export EMAIL_RCP=''
+
+# Load registry test credentials
+export REGISTRY_USER=''
+export REGISTRY_PWD=''
+
+cd docker/testing
+vagrant up --provider=aws
+
+
+github pull request
+===================

 The entire docker pull request test workflow is event driven by github. Its
 usage is fully automatic and the results are logged in docker-ci.dotcloud.com
@@ -13,3 +46,11 @@ buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it
 can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0)
 implemented this capability. Also we added a new scheduler to exclusively filter
 PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it.
+
+
+nighthly release
+================
+
+The nightly release process is done by buildbot, running a DinD container that downloads
+the docker repository and builds the release container. The resulting docker
+binary is then tested, and if everything is fine, the release is done.
--- a/hack/infrastructure/docker-ci/Dockerfile
+++ b/hack/infrastructure/docker-ci/Dockerfile
@@ -0,0 +1,43 @@
+# VERSION:        0.22
+# DOCKER-VERSION  0.6.3
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# DESCRIPTION:    Deploy docker-ci on Amazon EC2
+# COMMENTS:
+#     CONFIG_JSON is an environment variable json string loaded as:
+#
+# export CONFIG_JSON='
+#     { "AWS_TAG":             "EC2_instance_name",
+#       "AWS_ACCESS_KEY":      "EC2_access_key",
+#       "AWS_SECRET_KEY":      "EC2_secret_key",
+#       "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
+#       "DOCKER_CI_KEY":       "$(cat docker-ci_ssh_private_key.key)",
+#       "BUILDBOT_PWD":        "Buildbot_server_password",
+#       "IRC_PWD":             "Buildbot_IRC_password",
+#       "SMTP_USER":           "SMTP_server_user",
+#       "SMTP_PWD":            "SMTP_server_password",
+#       "PKG_ACCESS_KEY":      "Docker_release_S3_bucket_access_key",
+#       "PKG_SECRET_KEY":      "Docker_release_S3_bucket_secret_key",
+#       "PKG_GPG_PASSPHRASE":  "Docker_release_gpg_passphrase",
+#       "INDEX_AUTH":          "Index_encripted_user_password",
+#       "REGISTRY_USER":       "Registry_test_user",
+#       "REGISTRY_PWD":        "Registry_test_password",
+#       "REGISTRY_BUCKET":     "Registry_S3_bucket_name",
+#       "REGISTRY_ACCESS_KEY": "Registry_S3_bucket_access_key",
+#       "REGISTRY_SECRET_KEY": "Registry_S3_bucket_secret_key",
+#       "IRC_CHANNEL":         "Buildbot_IRC_channel",
+#       "EMAIL_RCP":           "Buildbot_mailing_receipient" }'
+#
+#
+# TO_BUILD:   docker build -t docker-ci .
+# TO_DEPLOY:  docker run -e CONFIG_JSON="${CONFIG_JSON}" docker-ci
+
+from ubuntu:12.04
+
+run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
+run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim
+run pip install boto fabric
+
+# Add deployment code and set default container command
+add . /docker-ci
+cmd "/docker-ci/deployment.py"
+
--- a/hack/infrastructure/docker-ci/MAINTAINERS
+++ b/hack/infrastructure/docker-ci/MAINTAINERS
--- a/hack/infrastructure/docker-ci/README.rst
+++ b/hack/infrastructure/docker-ci/README.rst
@@ -0,0 +1,26 @@
+=======
+testing
+=======
+
+This directory contains docker-ci testing related files.
+
+
+Buildbot
+========
+
+Buildbot is a continuous integration system designed to automate the
+build/test cycle. By automatically rebuilding and testing the tree each time
+something has changed, build problems are pinpointed quickly, before other
+developers are inconvenienced by the failure.
+
+We are running buildbot in Amazon's EC2 to verify docker passes all
+tests when commits get pushed to the master branch and building
+nightly releases using Docker in Docker awesome implementation made
+by Jerome Petazzoni.
+
+https://github.com/jpetazzo/dind
+
+Docker's buildbot instance is at http://docker-ci.dotcloud.com/waterfall
+
+For deployment instructions, please take a look at
+hack/infrastructure/docker-ci/Dockerfile
--- a/hack/infrastructure/docker-ci/buildbot/README.rst
+++ b/hack/infrastructure/docker-ci/buildbot/README.rst
@@ -0,0 +1 @@
+Buildbot configuration and setup files
--- a/hack/infrastructure/docker-ci/buildbot/buildbot.conf
+++ b/hack/infrastructure/docker-ci/buildbot/buildbot.conf
--- a/hack/infrastructure/docker-ci/buildbot/github.py
+++ b/hack/infrastructure/docker-ci/buildbot/github.py
@@ -86,10 +86,14 @@ def getChanges(request, options = None):
                the http request object
        """
        payload = json.loads(request.args['payload'][0])
+        import urllib,datetime
+        fname = str(datetime.datetime.now()).replace(' ','_').replace(':','-')[:19]
+        open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2))
+
 	if 'pull_request' in payload:
-	    user = payload['repository']['owner']['login']
-	    repo = payload['repository']['name']
-            repo_url = payload['repository']['html_url']
+	    user = payload['pull_request']['user']['login']
+	    repo = payload['pull_request']['head']['repo']['name']
+	    repo_url = payload['pull_request']['head']['repo']['html_url']
 	else:
 	    user = payload['repository']['owner']['name']
            repo = payload['repository']['name']
@@ -133,7 +137,7 @@ def process_change(payload, user, repo, repo_url, project):
 	    if 'pull_request' in payload:
 		changes = [{
 		    'category'   : 'github_pullrequest',
-                    'who'        : user,
+                    'who'        : '{0} - PR#{1}'.format(user,payload['number']),
                    'files'      : [],
                    'comments'   : payload['pull_request']['title'], 
                    'revision'   : newrev,
--- a/hack/infrastructure/docker-ci/buildbot/master.cfg
+++ b/hack/infrastructure/docker-ci/buildbot/master.cfg
@@ -20,7 +20,8 @@ TEST_PWD = 'docker'     # Credential to authenticate build triggers
 BUILDER_NAME = 'docker'
 GITHUB_DOCKER = 'github.com/dotcloud/docker'
 BUILDBOT_PATH = '/data/buildbot'
-DOCKER_PATH = '/data/docker'
+DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
+DOCKER_CI_PATH = '/docker-ci'
 BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
 PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'

@@ -45,49 +46,41 @@ c['slavePortnum'] = PORT_MASTER

 # Schedulers
 c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME,
-    'index','registry','coverage'])]
+    'index','registry','coverage','nightlyrelease'])]
 c['schedulers'] += [SingleBranchScheduler(name="all",
    change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None,
    builderNames=[BUILDER_NAME])]
 c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
    change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
    builderNames=['pullrequest'])]
-c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'],
-    hour=0, minute=30)]
+c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease'],
+    hour=7, minute=00)]
 c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
    hour=range(0,24,4), minute=15)]

 # Builders
 # Docker commit test
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True,
-    command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
-    "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
-    "git reset --hard %(src::revision)s; go test -v".format(
-    BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
-c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
+factory.addStep(ShellCommand(description='Docker', logEnviron=False,
+    usePTY=True, command=['sh', '-c', Interpolate(
+    '{0}/docker-test/test_docker.sh %(src::revision)s'.format(DOCKER_CI_PATH))]))
+c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'],
    factory=factory)]

 # Docker pull request test
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True,
-    command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
-    "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
-    "git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; "
-    "git checkout %(src::revision)s; git rebase master; go test -v".format(
-    PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
+factory.addStep(ShellCommand(description='pull_request', logEnviron=False,
+    usePTY=True, command=['sh', '-c', Interpolate(
+    '{0}/docker-test/test_docker.sh %(src::revision)s %(src::repository)s'
+    ' %(src::branch)s'.format(DOCKER_CI_PATH))]))
 c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
    factory=factory)]

 # Docker coverage test
-coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n'
-    'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n'
-    'sudo -E GOPATH=`pwd` ./bin/gocov test -deps -exclude-goroot -v'
-    ' -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,'
-    'code.google.com/p/go.net/websocket github.com/dotcloud/docker | ./bin/gocov report')
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True,
-    command=coverage_cmd))
+factory.addStep(ShellCommand(description='Coverage', logEnviron=False,
+    usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
+    DOCKER_CI_PATH)))
 c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
    factory=factory)]

@@ -95,8 +88,8 @@ c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='registry', logEnviron=False,
    command='. {0}/master/credentials.cfg; '
-    '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH,
-    DOCKER_PATH), usePTY=True))
+    '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH),
+    usePTY=True))
 c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
    factory=factory)]

@@ -109,6 +102,14 @@ factory.addStep(ShellCommand(description='index', logEnviron=False,
 c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
    factory=factory)]

+# Docker nightly release
+nightlyrelease_cmd = ('docker run -i -t -privileged -lxc-conf=lxc.aa_profile=unconfined'
+    ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,usePTY=True,
+    command=nightlyrelease_cmd))
+c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
+    factory=factory)]

 # Status
 authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]),
--- a/hack/infrastructure/docker-ci/buildbot/requirements.txt
+++ b/hack/infrastructure/docker-ci/buildbot/requirements.txt
--- a/hack/infrastructure/docker-ci/buildbot/setup.sh
+++ b/hack/infrastructure/docker-ci/buildbot/setup.sh
@@ -6,16 +6,22 @@

 USER=$1
 CFG_PATH=$2
-BUILDBOT_PWD=$3
-IRC_PWD=$4
-IRC_CHANNEL=$5
-SMTP_USER=$6
-SMTP_PWD=$7
-EMAIL_RCP=$8
+DOCKER_PATH=$3
+BUILDBOT_PWD=$4
+IRC_PWD=$5
+IRC_CHANNEL=$6
+SMTP_USER=$7
+SMTP_PWD=$8
+EMAIL_RCP=$9
+REGISTRY_USER=${10}
+REGISTRY_PWD=${11}
+REGISTRY_BUCKET=${12}
+REGISTRY_ACCESS_KEY=${13}
+REGISTRY_SECRET_KEY=${14}
 BUILDBOT_PATH="/data/buildbot"
-DOCKER_PATH="/data/docker"
 SLAVE_NAME="buildworker"
 SLAVE_SOCKET="localhost:9989"
+
 export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"

 function run { su $USER -c "$1"; }
@@ -35,6 +41,10 @@ run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg"
 run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
 run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
 run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
+run "echo 'export DOCKER_CREDS=\"$REGISTRY_USER:$REGISTRY_PWD\"' > $BUILDBOT_PATH/master/credentials.cfg"
+run "echo 'export S3_BUCKET=\"$REGISTRY_BUCKET\"' >> $BUILDBOT_PATH/master/credentials.cfg"
+run "echo 'export S3_ACCESS_KEY=\"$REGISTRY_ACCESS_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"
+run "echo 'export S3_SECRET_KEY=\"$REGISTRY_SECRET_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"

 # Patch github webstatus to capture pull requests
 cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks
--- a/hack/infrastructure/docker-ci/deployment.py
+++ b/hack/infrastructure/docker-ci/deployment.py
@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+
+import os, sys, re, json, base64
+from boto.ec2.connection import EC2Connection
+from subprocess import call
+from fabric import api
+from fabric.api import cd, run, put, sudo
+from os import environ as env
+from time import sleep
+
+# Remove SSH private key as it needs more processing
+CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','',
+    env['CONFIG_JSON'], flags=re.DOTALL))
+
+# Populate environment variables
+for key in CONFIG:
+    env[key] = CONFIG[key]
+
+# Load SSH private key
+env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
+    env['CONFIG_JSON'],flags=re.DOTALL)
+
+
+AWS_TAG = env.get('AWS_TAG','docker-ci')
+AWS_KEY_NAME = 'dotcloud-dev'       # Same as CONFIG_JSON['DOCKER_CI_PUB']
+AWS_AMI = 'ami-d582d6bc'            # Ubuntu 13.04
+AWS_REGION = 'us-east-1'
+AWS_TYPE = 'm1.small'
+AWS_SEC_GROUPS = 'gateway'
+AWS_IMAGE_USER = 'ubuntu'
+DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
+DOCKER_CI_PATH = '/docker-ci'
+CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
+
+
+class AWS_EC2:
+    '''Amazon EC2'''
+    def __init__(self, access_key, secret_key):
+        '''Set default API parameters'''
+        self.handler = EC2Connection(access_key, secret_key)
+    def create_instance(self, tag, instance_type):
+        reservation = self.handler.run_instances(**instance_type)
+        instance = reservation.instances[0]
+        sleep(10)
+        while instance.state != 'running':
+            sleep(5)
+            instance.update()
+            print "Instance state: %s" % (instance.state)
+        instance.add_tag("Name",tag)
+        print "instance %s done!" % (instance.id)
+        return instance.ip_address
+    def get_instances(self):
+        return self.handler.get_all_instances()
+    def get_tags(self):
+        return dict([(i.instances[0].id, i.instances[0].tags['Name'])
+            for i in self.handler.get_all_instances() if i.instances[0].tags])
+    def del_instance(self, instance_id):
+        self.handler.terminate_instances(instance_ids=[instance_id])
+
+
+def json_fmt(data):
+    '''Format json output'''
+    return json.dumps(data, sort_keys = True, indent = 2)
+
+
+# Create EC2 API handler
+ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
+
+# Stop processing if AWS_TAG exists on EC2
+if AWS_TAG in ec2.get_tags().values():
+    print ('Instance: {} already deployed. Not further processing.'
+        .format(AWS_TAG))
+    exit(1)
+
+ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
+    'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
+
+# Wait 30 seconds for the machine to boot
+sleep(30)
+
+# Create docker-ci ssh private key so docker-ci docker container can communicate
+# with its EC2 instance
+os.makedirs('/root/.ssh')
+open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY'])
+os.chmod('/root/.ssh/id_rsa',0600)
+open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
+
+api.env.host_string = ip
+api.env.user = AWS_IMAGE_USER
+api.env.key_filename = '/root/.ssh/id_rsa'
+
+# Correct timezone
+sudo('echo "America/Los_Angeles" >/etc/timezone')
+sudo('dpkg-reconfigure --frontend noninteractive tzdata')
+
+# Load public docker-ci key
+sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB']))
+
+# Create docker nightly release credentials file
+credentials = {
+    'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'],
+    'AWS_SECRET_KEY': env['PKG_SECRET_KEY'],
+    'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'],
+    'INDEX_AUTH': env['INDEX_AUTH']}
+open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
+    base64.b64encode(json.dumps(credentials)))
+
+# Transfer docker
+sudo('mkdir -p ' + DOCKER_CI_PATH)
+sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
+call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
+    os.path.dirname(DOCKER_CI_PATH)), shell=True)
+
+# Install Docker and Buildbot dependencies
+sudo('addgroup docker')
+sudo('usermod -a -G docker ubuntu')
+sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
+sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
+sudo('echo deb https://get.docker.io/ubuntu docker main >'
+    ' /etc/apt/sources.list.d/docker.list')
+sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
+    'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"'
+    ' > /etc/apt/sources.list; apt-get update')
+sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
+    ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
+    ' aufs-tools make libfontconfig libevent-dev')
+sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
+    'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
+sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
+sudo('pip install -r {}/requirements.txt'.format(CFG_PATH))
+
+# Install docker and testing dependencies
+sudo('apt-get install -y -q lxc-docker')
+sudo('curl -s https://phantomjs.googlecode.com/files/'
+    'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin'
+    ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs')
+
+#### FIXME. Temporarily install docker with proper apparmor handling
+sudo('stop docker')
+sudo('wget -q -O /usr/bin/docker http://test.docker.io/test/docker')
+sudo('start docker')
+
+# Build docker-ci containers
+sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
+sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
+    DOCKER_CI_PATH))
+
+# Setup buildbot
+sudo('mkdir /data')
+sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'
+    ' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'],
+    env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'],
+    env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'],
+    env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'],
+    env['REGISTRY_SECRET_KEY']))
--- a/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh
+++ b/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -x
+# Generate a random string of $1 characters
+function random {
+    cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
+}
+
+# Compute test paths
+BASE_PATH=`pwd`/test_docker_$(random 12)
+DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
+export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
+
+# Fetch latest master
+mkdir -p $DOCKER_PATH
+cd $DOCKER_PATH
+git init .
+git fetch -q http://github.com/dotcloud/docker master
+git reset --hard FETCH_HEAD
+
+# Fetch go coverage
+cd $BASE_PATH/go
+GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov
+sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\
+ -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\
+code.google.com/p/go.net/websocket,github.com/dotcloud/tar\
+ github.com/dotcloud/docker | ./bin/gocov report; exit_status=$?
+
+# Cleanup testing directory
+rm -rf $BASE_PATH
+
+exit $exit_status
--- a/hack/infrastructure/docker-ci/docker-test/test_docker.sh
+++ b/hack/infrastructure/docker-ci/docker-test/test_docker.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -x
+COMMIT=${1-HEAD}
+REPO=${2-http://github.com/dotcloud/docker}
+BRANCH=${3-master}
+
+# Generate a random string of $1 characters
+function random {
+    cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
+}
+
+# Compute test paths
+BASE_PATH=`pwd`/test_docker_$(random 12)
+DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
+export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
+
+# Fetch latest master
+mkdir -p $DOCKER_PATH
+cd $DOCKER_PATH
+git init .
+git fetch -q http://github.com/dotcloud/docker master
+git reset --hard FETCH_HEAD
+
+# Merge commit
+git fetch -q "$REPO" "$BRANCH"
+git merge --no-edit $COMMIT || exit 1
+
+# Test commit
+go test -v; exit_status=$?
+
+# Cleanup testing directory
+rm -rf $BASE_PATH
+
+exit $exit_status
--- a/hack/infrastructure/docker-ci/functionaltests/test_index.py
+++ b/hack/infrastructure/docker-ci/functionaltests/test_index.py
--- a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh
+++ b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -x
+
+# Cleanup
+rm -rf docker-registry
+
+# Setup the environment
+export SETTINGS_FLAVOR=test
+export DOCKER_REGISTRY_CONFIG=config_test.yml
+
+# Get latest docker registry
+git clone -q https://github.com/dotcloud/docker-registry.git
+cd docker-registry
+
+# Get dependencies
+pip install -q -r requirements.txt
+pip install -q -r test-requirements.txt
+pip install -q tox
+
+# Run registry tests
+tox || exit 1
+export PYTHONPATH=$(pwd)/docker-registry
+python -m unittest discover -p s3.py -s test || exit 1
+python -m unittest discover -p workflow.py -s test
+
--- a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile
+++ b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile
@@ -0,0 +1,37 @@
+# VERSION:        1.2
+# DOCKER-VERSION  0.6.3
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# DESCRIPTION:    Build docker nightly release using Docker in Docker.
+# REFERENCES:     This code reuses the excellent implementation of docker in docker
+#                 made by Jerome Petazzoni.  https://github.com/jpetazzo/dind
+# COMMENTS:
+#   release_credentials.json is a base64 json encoded file containing:
+#       { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id",
+#         "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key'
+#         "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature'
+#         "INDEX_AUTH='Encripted_index_authentication' }
+# TO_BUILD:       docker build -t dockerbuilder .
+# TO_RELEASE:     docker run -i -t -privileged -lxc-conf="lxc.aa_profile = unconfined" -e AWS_S3_BUCKET="test.docker.io" dockerbuilder
+
+from docker
+maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+
+# Add docker dependencies and downloading packages
+run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
+run apt-get update; apt-get install -y -q wget python2.7
+
+# Add production docker binary
+run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker
+
+#### FIXME. Temporarily install docker with proper apparmor handling
+run wget -q -O /usr/bin/docker http://test.docker.io/test/docker; chmod +x /usr/bin/docker
+
+# Add proto docker builder
+add ./dockerbuild /usr/bin/dockerbuild
+run chmod +x /usr/bin/dockerbuild
+
+# Add release credentials
+add ./release_credentials.json /root/release_credentials.json
+
+# Launch build process in a container
+cmd dockerbuild
--- a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild
+++ b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH
+# are decoded from /root/release_credentials.json
+# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
+
+# Enable debugging
+set -x
+
+# Fetch docker master branch
+rm -rf  /go/src/github.com/dotcloud/docker
+cd /
+git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
+cd /go/src/github.com/dotcloud/docker
+
+echo FIXME. Temporarily add Jerome changeset with proper apparmor handling
+git fetch  http://github.com/jpetazzo/docker escape-apparmor-confinement:escape-apparmor-confinement
+git rebase --onto master master escape-apparmor-confinement
+
+# Launch docker daemon using dind inside the container
+./hack/dind /usr/bin/docker -d &
+sleep 5
+
+# Add an uncommitted change to generate a timestamped release
+date > timestamp
+
+# Build the docker package using /Dockerfile
+docker build -t docker .
+
+# Run Docker unittests binary and Ubuntu package
+docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh || exit 1
+
+# Turn debug off to load credentials from the environment
+set +x
+eval $(cat /root/release_credentials.json  | python -c '
+import sys,json,base64;
+d=json.loads(base64.b64decode(sys.stdin.read()));
+exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
+echo '{"https://index.docker.io/v1/":{"auth":"'$INDEX_AUTH'","email":"engineering@dotcloud.com"}}' > /.dockercfg
+set -x
+
+# Push docker nightly
+echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release  hack/release.sh
+set +x
+docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE  release  hack/release.sh
--- a/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json
+++ b/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json
@@ -0,0 +1 @@
+eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0=
--- a/testing/README.rst
+++ b/testing/README.rst
@@ -1,58 +0,0 @@
-=======
-testing
-=======
-
-This directory contains testing related files.
-
-
-Buildbot
-========
-
-Buildbot is a continuous integration system designed to automate the
-build/test cycle. By automatically rebuilding and testing the tree each time
-something has changed, build problems are pinpointed quickly, before other
-developers are inconvenienced by the failure.
-
-We are running buildbot in an AWS instance to verify docker passes all tests
-when commits get pushed to the master branch.
-
-You can check docker's buildbot instance at http://docker-ci.dotcloud.com/waterfall
-
-
-Deployment
-~~~~~~~~~~
-
-::
-
-  # Define AWS credential environment variables
-  export AWS_ACCESS_KEY_ID=xxxxxxxxxxxx
-  export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxx
-  export AWS_KEYPAIR_NAME=xxxxxxxxxxxx
-  export AWS_SSH_PRIVKEY=xxxxxxxxxxxx
-
-  # Define email recipient and IRC channel
-  export EMAIL_RCP=xxxxxx@domain.com
-  export IRC_CHANNEL=docker
-
-  # Define buildbot credentials
-  export BUILDBOT_PWD=xxxxxxxxxxxx
-  export IRC_PWD=xxxxxxxxxxxx
-  export SMTP_USER=xxxxxxxxxxxx
-  export SMTP_PWD=xxxxxxxxxxxx
-
-  # Define docker registry functional test credentials
-  export REGISTRY_USER=xxxxxxxxxxxx
-  export REGISTRY_PWD=xxxxxxxxxxxx
-
-  # Checkout docker
-  git clone git://github.com/dotcloud/docker.git
-
-  # Deploy docker on AWS
-  cd docker/testing
-  vagrant up --provider=aws
-
-
-Buildbot AWS dependencies
-------------------------
-
-vagrant, virtualbox packages and vagrant aws plugin
--- a/testing/Vagrantfile
+++ b/testing/Vagrantfile
@@ -1,74 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-BOX_NAME = "docker-ci"
-BOX_URI = "http://cloud-images.ubuntu.com/vagrant/raring/current/raring-server-cloudimg-amd64-vagrant-disk1.box"
-AWS_AMI = "ami-10314d79"
-DOCKER_PATH = "/data/docker"
-CFG_PATH = "#{DOCKER_PATH}/testing/buildbot"
-on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \
-  Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \
-  (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox)
-USER = on_vbox ? "vagrant": "ubuntu"
-
-Vagrant::Config.run do |config|
-  # Setup virtual machine box. This VM configuration code is always executed.
-  config.vm.box = BOX_NAME
-  config.vm.box_url = BOX_URI
-  config.vm.forward_port 8010, 8010
-  config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
-
-
-  # Deploy buildbot and its dependencies if it was not done
-  if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
-    # Add memory limitation capabilities
-    pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; '
-    # Adjust kernel
-    pkg_cmd << "apt-get update -qq; "
-    if on_vbox
-      pkg_cmd << "apt-get install -q -y linux-image-extra-`uname -r`; "
-    else
-      pkg_cmd << "apt-get install -q -y linux-image-generic; "
-    end
-
-    # Deploy buildbot CI
-    pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \
-      "pip install -r #{CFG_PATH}/requirements.txt; " \
-      "chown #{USER}.#{USER} /data; cd /data; " \
-      "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \
-        "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \
-        "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \
-      "#{CFG_PATH}/setup_credentials.sh #{USER} " \
-        "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; "
-    # Install docker and testing dependencies
-    pkg_cmd << "curl -s https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | " \
-      "  tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go; " \
-      "curl -s https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 | " \
-      "  tar jx -C /usr/bin --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs; " \
-      "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial aufs-tools make libfontconfig; " \
-      "export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \
-      "rm -rf ${GOPATH}/src/github.com/dotcloud/docker; "
-    # Activate new kernel options
-    pkg_cmd << "shutdown -r +1; "
-    config.vm.provision :shell, :inline => pkg_cmd
-  end
-end
-
-# Providers were added on Vagrant >= 1.1.0
-Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
-  config.vm.provider :aws do |aws, override|
-    aws.tags = { 'Name' => 'docker-ci' }
-    aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
-    aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
-    aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
-    override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"]
-    override.ssh.username = USER
-    aws.ami = AWS_AMI
-    aws.region = "us-east-1"
-    aws.instance_type = "m1.small"
-    aws.security_groups = "gateway"
-  end
-
-  config.vm.provider :virtualbox do |vb|
-  end
-end
--- a/testing/buildbot/README.rst
+++ b/testing/buildbot/README.rst
@@ -1 +0,0 @@
-Buildbot configuration and setup files (except Vagrantfile located on ..)
--- a/testing/buildbot/credentials.cfg
+++ b/testing/buildbot/credentials.cfg
@@ -1,5 +0,0 @@
-# Credentials for tests. Buildbot source this file on tests
-# when needed.
-
-# Docker registry credentials. Format: 'username:password'
-export DOCKER_CREDS=''
--- a/testing/buildbot/setup_credentials.sh
+++ b/testing/buildbot/setup_credentials.sh
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# Setup of test credentials. Called by Vagrantfile
-export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
-
-USER=$1
-REGISTRY_USER=$2
-REGISTRY_PWD=$3
-
-BUILDBOT_PATH="/data/buildbot"
-DOCKER_PATH="/data/docker"
-
-function run { su $USER -c "$1"; }
-
-run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master"
-cd $BUILDBOT_PATH/master
-run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg"
--- a/testing/functionaltests/test_registry.sh
+++ b/testing/functionaltests/test_registry.sh
@@ -1,11 +0,0 @@
-#!/bin/sh
-
-# Cleanup
-rm -rf docker-registry
-
-# Get latest docker registry
-git clone https://github.com/dotcloud/docker-registry.git
-
-# Configure and run registry tests
-cd docker-registry; cp config_sample.yml config.yml
-cd test; python -m unittest workflow
				`@@ -0,0 +1 @@`
				`eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0=`
				`@@ -1 +0,0 @@`
				`Buildbot configuration and setup files (except Vagrantfile located on ..)`