add docker-compose.yml creating a swift proxy with our middleware. At the same...

add docker-compose.yml creating a swift proxy with our middleware. At the same time create kafka/zookeeper cluster with kafka-manager webui.

docker/docker-compose.yml

+version: '2'
+services:
+  swift:
+    build:
+      context: ..
+      dockerfile: docker/swift/Dockerfile
+    ports:
+      - "12345:8080"
+    volumes:
+      - /srv/swift:/srv
+    links:
+      - zookeeper
+      - kafka
+
+  kafka:
+    image: wurstmeister/kafka
+    ports:
+      - "9092:9092"
+    links:
+      - zookeeper
+    environment:
+      JMX_PORT: 7000
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CREATE_TOPICS: "swift-messagebus:1:1"
+      KAFKA_ADVERTISED_PORT: 9092
+
+  # kafka (at least this image) needs zookeeper
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      JMX_PORT: 7000
+
+  # some webui is always good, attach it and connectit tozookeeper:2181
+  kafka-manager:
+    image: sheepkiller/kafka-manager
+    ports:
+      - "9000:9000"
+    links:
+      - zookeeper
+      - kafka
+    environment:
+      ZK_HOSTS: zookeeper:2181
+      APPLICATION_SECRET: s3cr3t

docker/swift/Dockerfile

+FROM ubuntu:14.04
+MAINTAINER bne86 <b.von.st.vieth@fz-juelich.de>
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository cloud-archive:liberty && \
+    apt-get update && \
+    apt-get install -y supervisor swift python-swiftclient rsync \
+                       swift-proxy swift-object memcached python-keystoneclient \
+                       python-swiftclient swift-plugin-s3 python-netifaces \
+                       python-xattr python-memcache \
+                       swift-account swift-container swift-object pwgen
+
+RUN mkdir -p /var/log/supervisor
+ADD docker/swift/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+#
+# Swift configuration
+# - Partially fom http://docs.openstack.org/developer/swift/development_saio.html
+#
+
+# not sure how valuable dispersion will be...
+ADD docker/swift/dispersion.conf /etc/swift/dispersion.conf
+ADD docker/swift/rsyncd.conf /etc/rsyncd.conf
+ADD docker/swift/swift.conf /etc/swift/swift.conf
+ADD docker/swift/proxy-server.conf /etc/swift/proxy-server.conf
+ADD docker/swift/account-server.conf /etc/swift/account-server.conf
+ADD docker/swift/object-server.conf /etc/swift/object-server.conf
+ADD docker/swift/container-server.conf /etc/swift/container-server.conf
+ADD docker/swift/startmain.sh /usr/local/bin/startmain.sh
+RUN chmod 755 /usr/local/bin/startmain.sh
+
+ADD . swift-kafka
+WORKDIR swift-kafka
+RUN python setup.py install
+
+EXPOSE 8080
+
+CMD /usr/local/bin/startmain.sh

docker/swift/account-server.conf

+[DEFAULT]
+devices = /srv/node
+mount_check = false
+disable_fallocate = true
+bind_port = 6012
+workers = 1
+user = swift
+log_facility = LOG_LOCAL2
+recon_cache_path = /var/cache/swift
+eventlet_debug = true
+
+[pipeline:main]
+pipeline = recon account-server
+
+[app:account-server]
+use = egg:swift#account
+
+[filter:recon]
+use = egg:swift#recon
+
+[account-replicator]
+vm_test_mode = yes
+
+[account-auditor]
+
+[account-reaper]

docker/swift/container-server.conf

+[DEFAULT]
+devices = /srv/node
+mount_check = false
+disable_fallocate = true
+bind_port = 6011
+workers = 1
+user = swift
+log_facility = LOG_LOCAL2
+recon_cache_path = /var/cache/swift
+eventlet_debug = true
+allow_versions = true
+
+[pipeline:main]
+pipeline = recon container-server
+
+[app:container-server]
+use = egg:swift#container
+
+[filter:recon]
+use = egg:swift#recon
+
+[container-replicator]
+vm_test_mode = yes
+
+[container-updater]
+
+[container-auditor]
+
+[container-sync]

docker/swift/dispersion.conf

+[dispersion]
+auth_url = http://127.0.0.1:8080/auth/v1.0
+auth_user = test:tester
+auth_key = testing
+endpoint_type = internalURL
\ No newline at end of file

docker/swift/object-expirer.conf

+[DEFAULT]
+# swift_dir = /etc/swift
+user = swift
+# You can specify default log routing here if you want:
+log_name = object-expirer
+log_facility = LOG_LOCAL6
+log_level = INFO
+#log_address = /dev/log
+#
+# comma separated list of functions to call to setup custom log handlers.
+# functions get passed: conf, name, log_to_console, log_route, fmt, logger,
+# adapted_logger
+# log_custom_handlers =
+#
+# If set, log_udp_host will override log_address
+# log_udp_host =
+# log_udp_port = 514
+#
+# You can enable StatsD logging here:
+# log_statsd_host = localhost
+# log_statsd_port = 8125
+# log_statsd_default_sample_rate = 1.0
+# log_statsd_sample_rate_factor = 1.0
+# log_statsd_metric_prefix =
+
+[object-expirer]
+interval = 300
+# auto_create_account_prefix = .
+# report_interval = 300
+# concurrency is the level of concurrency o use to do the work, this value
+# must be set to at least 1
+# concurrency = 1
+# processes is how many parts to divide the work into, one part per process
+#   that will be doing the work
+# processes set 0 means that a single process will be doing all the work
+# processes can also be specified on the command line and will override the
+#   config value
+# processes = 0
+# process is which of the parts a particular process will work on
+# process can also be specified on the command line and will overide the config
+#   value
+# process is "zero based", if you want to use 3 processes, you should run
+#  processes with process set to 0, 1, and 2
+# process = 0
+
+[pipeline:main]
+pipeline = catch_errors cache proxy-server
+
+[app:proxy-server]
+use = egg:swift#proxy
+# See proxy-server.conf-sample for options
+
+[filter:cache]
+use = egg:swift#memcache
+# See proxy-server.conf-sample for options
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+# See proxy-server.conf-sample for options
\ No newline at end of file

docker/swift/object-server.conf

+[DEFAULT]
+devices = /srv/node
+mount_check = false
+disable_fallocate = true
+bind_port = 6010
+workers = 1
+user = swift
+log_facility = LOG_LOCAL2
+recon_cache_path = /var/cache/swift
+eventlet_debug = true
+
+[pipeline:main]
+pipeline = recon object-server
+
+[app:object-server]
+use = egg:swift#object
+
+[filter:recon]
+use = egg:swift#recon
+
+[object-replicator]
+vm_test_mode = yes
+
+[object-updater]
+
+[object-auditor]

docker/swift/proxy-server.conf

+[DEFAULT]
+bind_port = 8080
+workers = 1
+user = swift
+log_facility = LOG_LOCAL1
+eventlet_debug = true
+
+[pipeline:main]
+# Yes, proxy-logging appears twice. This is so that
+# middleware-originated requests get logged too.
+pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl slo dlo ratelimit crossdomain tempauth staticweb container-quotas account-quotas messagebus proxy-logging proxy-server
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+
+[filter:healthcheck]
+use = egg:swift#healthcheck
+
+[filter:proxy-logging]
+use = egg:swift#proxy_logging
+
+[filter:bulk]
+use = egg:swift#bulk
+
+[filter:ratelimit]
+use = egg:swift#ratelimit
+
+[filter:crossdomain]
+use = egg:swift#crossdomain
+
+[filter:dlo]
+use = egg:swift#dlo
+
+[filter:slo]
+use = egg:swift#slo
+
+[filter:tempurl]
+use = egg:swift#tempurl
+
+[filter:tempauth]
+storage_url_scheme = default
+use = egg:swift#tempauth
+user_admin_admin = admin .admin .reseller_admin
+user_test_tester = testing .admin
+user_test2_tester2 = testing2 .admin
+user_test_tester3 = testing3
+
+[filter:staticweb]
+use = egg:swift#staticweb
+
+[filter:account-quotas]
+use = egg:swift#account_quotas
+
+[filter:container-quotas]
+use = egg:swift#container_quotas
+
+[filter:cache]
+use = egg:swift#memcache
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+
+[filter:messagebus]
+use = egg:messagebus#messagebus
+url = asd
+username = asd
+password = asd
+
+[app:proxy-server]
+use = egg:swift#proxy
+allow_account_management = true
+account_autocreate = true

docker/swift/rsyncd.conf

+uid = swift
+gid = swift
+log file = /var/log/rsyncd.log
+pid file = /var/run/rsyncd.pid
+address = 127.0.0.1
+
+[account6012]
+max connections = 25
+path = /srv/node
+read only = false
+lock file = /var/lock/account6012.lock
+
+[container6011]
+max connections = 25
+path = /srv/node
+read only = false
+lock file = /var/lock/container6011.lock
+
+[object6010]
+max connections = 25
+path = /srv/node
+read only = false
+lock file = /var/lock/object6010.lock

docker/swift/startmain.sh

+#!/bin/bash
+
+#
+# Make the rings if they don't exist already
+#
+
+# These can be set with docker run -e VARIABLE=X at runtime
+SWIFT_PART_POWER=${SWIFT_PART_POWER:-7}
+SWIFT_PART_HOURS=${SWIFT_PART_HOURS:-1}
+SWIFT_REPLICAS=${SWIFT_REPLICAS:-1}
+
+if [ -e /srv/account.builder ]; then
+	echo "Ring files already exist in /srv, copying them to /etc/swift..."
+	cp /srv/*.builder /etc/swift/
+	cp /srv/*.gz /etc/swift/
+fi
+
+# This comes from a volume, so need to chown it here, not sure of a better way
+# to get it owned by Swift.
+if [ ! -e /srv/node ]; then
+  mkdir /srv/node
+fi
+chown -R swift:swift /srv
+
+if [ ! -e /etc/swift/account.builder ]; then
+
+	cd /etc/swift
+
+	# 2^& = 128 we are assuming just one drive
+	# 1 replica only
+
+	echo "No existing ring files, creating them..."
+
+	swift-ring-builder object.builder create ${SWIFT_PART_POWER} ${SWIFT_REPLICAS} ${SWIFT_PART_HOURS}
+	swift-ring-builder object.builder add r1z1-127.0.0.1:6010/sdb1 1
+	swift-ring-builder object.builder rebalance
+	swift-ring-builder container.builder create ${SWIFT_PART_POWER} ${SWIFT_REPLICAS} ${SWIFT_PART_HOURS}
+	swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1
+	swift-ring-builder container.builder rebalance
+	swift-ring-builder account.builder create ${SWIFT_PART_POWER} ${SWIFT_REPLICAS} ${SWIFT_PART_HOURS}
+	swift-ring-builder account.builder add r1z1-127.0.0.1:6012/sdb1 1
+	swift-ring-builder account.builder rebalance
+
+	# Back these up for later use
+	echo "Copying ring files to /srv to save them if it's a docker volume..."
+	cp *.gz /srv
+	cp *.builder /srv
+
+fi
+
+# If you are going to put an ssl terminator in front of the proxy, then I believe
+# the storage_url_scheme should be set to https. So if this var isn't empty, set
+# the default storage url to https.
+if [ ! -z "${SWIFT_STORAGE_URL_SCHEME}" ]; then
+	echo "Setting default_storage_scheme to https in proxy-server.conf..."
+	sed -i -e "s/storage_url_scheme = default/storage_url_scheme = https/g" /etc/swift/proxy-server.conf
+	grep "storage_url_scheme" /etc/swift/proxy-server.conf
+fi
+
+if [ ! -z "${SWIFT_USER_PASSWORD}" ]; then
+	echo "Setting passwords in /etc/swift/proxy-server.conf"
+	PASS=`pwgen 12 1`
+	sed -i -e "s/user_admin_admin = admin .admin .reseller_admin/user_admin_admin = ${SWIFT_USER_PASSWORD} .admin .reseller_admin/g" /etc/swift/proxy-server.conf
+	sed -i -e "s/user_test_tester = testing .admin/user_test_tester = ${SWIFT_USER_PASSWORD} .admin/g" /etc/swift/proxy-server.conf
+	sed -i -e "s/user_test2_tester2 = testing2 .admin/user_test2_tester2 = ${SWIFT_USER_PASSWORD} .admin/g" /etc/swift/proxy-server.conf
+	sed -i -e "s/user_test_tester3 = testing3/user_test_tester3 = ${SWIFT_USER_PASSWORD}/g" /etc/swift/proxy-server.conf
+	grep "user_test" /etc/swift/proxy-server.conf
+fi
+
+# Start supervisord
+echo "Starting supervisord..."
+/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+
+#
+# Tail the log file for "docker log $CONTAINER_ID"
+#
+
+# sleep waiting for rsyslog to come up under supervisord
+sleep 3
+
+echo "Starting to tail /var/log/syslog...(hit ctrl-c if you are starting the container in a bash shell)"
+
+tail -n 0 -f /var/log/syslog

docker/swift/supervisord.conf

+[supervisord]
+nodaemon=false
+
+[program:memcached]
+command=/usr/bin/memcached -u memcache
+startsecs=3
+stopwaitsecs = 3
+
+[program:rsyslog]
+command=/bin/bash -c "source /etc/default/rsyslog && /usr/sbin/rsyslogd -n -c3"
+startsecs = 5
+stopwaitsecs = 5
+
+[program:proxy-server]
+command=/usr/bin/python /usr/bin/swift-proxy-server /etc/swift/proxy-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:container-server]
+command=/usr/bin/python /usr/bin/swift-container-server /etc/swift/container-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:account-server]
+command=/usr/bin/python /usr/bin/swift-account-server /etc/swift/account-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:account-auditor]
+command=/usr/bin/python /usr/bin/swift-account-auditor /etc/swift/account-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:object-replicator]
+command=/usr/bin/python /usr/bin/swift-object-replicator /etc/swift/object-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:object-auditor]
+command=/usr/bin/python /usr/bin/swift-object-auditor /etc/swift/object-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[progam:container-auditor]
+command=/usr/bin/python /usr/bin/swift-container-auditor /etc/swift/container-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:object-auditor]
+command=/usr/bin/python /usr/bin/swift-object-server /etc/swift/object-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:account-replicator]
+command=/usr/bin/python /usr/bin/swift-account-replicator /etc/swift/account-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:account-reaper]
+command=/usr/bin/python /usr/bin/swift-account-reaper /etc/swift/account-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:container-sync]
+command=/usr/bin/python /usr/bin/swift-container-sync /etc/swift/container-server.conf
+startsecs=3
+stopwaitsecs = 3
+
+[program:object-updater]
+command=/usr/bin/python /usr/bin/swift-object-updater /etc/swift/object-server.conf
+startsecs=3
+stopwaitsecs = 3
+

docker/swift/swift.conf

+[swift-hash]
+# random unique strings that can never change (DO NOT LOSE)
+swift_hash_path_prefix = changeme
+swift_hash_path_suffix = changeme
\ No newline at end of file