koding · cihangir · Oct 23, 2017 · cihangir · Oct 23, 2017 · mertaytore
diff --git a/config/generateRunFile.coffee b/config/generateRunFile.coffee
      export CHANGE_MINIKUBE_NONE_USER=true
      sudo -E minikube start --vm-driver=none

      sleep 60

+      envsubst < deployment/kubernetes/build-pod.yaml.template > deployment/kubernetes/build-pod.yaml
diff --git a/config/main.dev.coffee b/config/main.dev.coffee
  KONFIG.supervisorConf = (require '../deployment/supervisord.coffee').create KONFIG
+  KONFIG.kubernetesConf = (require '../deployment/kubernetes.coffee').create KONFIG
+  for name, options of KONFIG.workers when options.kubernetes?.ports?
+    fs.writeFileSync "./deployment/kubernetes/backend-pod/#{name}-svc.yaml", (require '../deployment/kubernetes.coffee').createWorkerServices name, options
diff --git a/deployment/kubernetes.coffee b/deployment/kubernetes.coffee
+  return mountConf
+
+
+generateVolumesSection = ->
diff --git a/config/workers.coffee b/config/workers.coffee
+      kubernetes        :
+        image           : 'nginx'
+        command         : " [ \"nginx\", \"-c\", \"#{KONFIG.projectRoot}/nginx.conf\" ] "
+        mounts          : [ 'koding-working-tree', 'assets' ]
diff --git a/config/workers.coffee b/config/workers.coffee
+        ports						:
+          containerPort : "#{KONFIG.kontrol.port}"
+          hostPort      : "#{KONFIG.kontrol.port}"
+        envVariables    : [
diff --git a/config/workers.coffee b/config/workers.coffee
@@ -116,13 +174,34 @@ module.exports = (KONFIG, options, credentials) ->
        ]
      healthCheckURLs   : [ "http://localhost:#{KONFIG.kloud.port}/healthCheck" ]
      versionURL        : "http://localhost:#{KONFIG.kloud.port}/version"
+      kubernetes        :
+        image           : 'koding/base'
+        command         : " [ \"./run\", \"exec\", \"#{GOBIN_K8S}/kloud\" ] "
diff --git a/config/workers.coffee b/config/workers.coffee
+        mounts          : [ 'koding-working-tree', 'root-kite-volume', 'generated-volume' ]
+        ports:
+          containerPort : "#{KONFIG.kloud.port}"
+          hostPort      : "#{KONFIG.kloud.port}"
diff --git a/config/workers.coffee b/config/workers.coffee
@@ -116,13 +174,34 @@ module.exports = (KONFIG, options, credentials) ->
        ]
      healthCheckURLs   : [ "http://localhost:#{KONFIG.kloud.port}/healthCheck" ]
      versionURL        : "http://localhost:#{KONFIG.kloud.port}/version"
+      kubernetes        :
+        image           : 'koding/base'
diff --git a/config/workers.coffee b/config/workers.coffee
+        mounts          : [ 'koding-working-tree', 'root-kite-volume' ]
+        ports						:
+          containerPort : '4560'
+          hostPort			: '4560'
diff --git a/deployment/kubernetes.coffee b/deployment/kubernetes.coffee
+
+  containerSection =
+  """
+  \n    - name: #{container.name}
diff --git a/deployment/nginx.coffee b/deployment/nginx.coffee
@@ -15,6 +15,10 @@ createUpstreams = (KONFIG) ->
      servers += '\n' if servers isnt ''
      port = parseInt(port, 10)

+      if name is 'webserver' and options.kubernetes.ports?
diff --git a/scripts/bootstrap-container b/scripts/bootstrap-container
@@ -41,6 +41,13 @@ function run() {
 	esac
 }

+function k8s_build() {
diff --git a/.gitignore b/.gitignore
+
+deployment/kubernetes/backend-pod/
+deployment/kubernetes/frontend-pod/client-containers.yaml
+deployment/kubernetes/build-pod.yaml
diff --git a/config/generateKonfig.coffee b/config/generateKonfig.coffee
@@ -255,7 +255,7 @@ module.exports = (options, credentials) ->
    # TODO: average request count per hour for a user should be measured and a reasonable limit should be set
    nodejsRateLimiter             : { enabled: no,  guestRules: [{ interval: 3600, limit: 5000 }], userRules: [{ interval: 3600, limit: 10000 }] } # limit: request limit per rate limit window, interval: rate limit window duration in seconds
    nodejsRateLimiterForApi       : { enabled: yes, guestRules: [{ interval: 60,   limit: 5 }],    userRules: [{ interval: 60,   limit: 60 }] }    # limit: request limit per rate limit window, interval: rate limit window duration in seconds
-    webserver                     : { port: 8080 }
+    webserver                     : { port: 8080, k8sPort: 8040 }
diff --git a/config/main.dev.coffee b/config/main.dev.coffee
@@ -105,7 +105,12 @@ Configuration = (options = {}) ->
    sh: (require './generateShellEnv').create KONFIG, options
    json: JSON.stringify KONFIG, null, 2

+  if not fs.existsSync './deployment/kubernetes/backend-pod' then fs.mkdirSync './deployment/kubernetes/backend-pod'
diff --git a/config/workers.coffee b/config/workers.coffee
@@ -5,14 +5,55 @@ module.exports = (KONFIG, options, credentials) ->
  GOBIN = '%(ENV_KONFIG_PROJECTROOT)s/go/bin'
  GOPATH = '%(ENV_KONFIG_PROJECTROOT)s/go'

+  GOBIN_K8S = "#{KONFIG.projectRoot}/go/bin"
+  GOPATH_K8S = "#{KONFIG.projectRoot}/go"
diff --git a/config/workers.coffee b/config/workers.coffee
+        stopsignal      : 'QUIT'
+      kubernetes        :
+        image           : 'nginx'
+        command         : " [ \"nginx\", \"-c\", \"#{KONFIG.projectRoot}/nginx.conf\" ] "
diff --git a/deployment/nginx.coffee b/deployment/nginx.coffee
@@ -15,6 +15,10 @@ createUpstreams = (KONFIG) ->
      servers += '\n' if servers isnt ''
      port = parseInt(port, 10)

+      if name is 'webserver' and options.kubernetes.ports?
+        webserverPort = parseInt(options.kubernetes.ports.containerPort, 10)
+        servers += "\tserver 127.0.0.1:#{webserverPort + index} max_fails=3 fail_timeout=10s;\n"
diff --git a/scripts/k8s-utilities.sh b/scripts/k8s-utilities.sh
+  # more info on guest user update: https://www.rabbitmq.com/blog/2014/04/02/breaking-things-with-rabbitmq-3-3/
+  k8s_health_check $RABBITMQ_POD_NAME koding 5 120 rabbitmq
+  sleep 10
+  kubectl exec -n koding $RABBITMQ_POD_NAME -c rabbitmq -- bash -c "rabbitmqctl add_user test test && rabbitmqctl set_user_tags test administrator && rabbitmqctl set_permissions -p / test '.*' '.*' '.*'"
diff --git a/scripts/k8s-utilities.sh b/scripts/k8s-utilities.sh
+else
+  echo "Unknown command: $1"
+  exit 1
+fi
diff --git a/scripts/k8s-utilities.sh b/scripts/k8s-utilities.sh
+  kubectl delete -f $1
+}
+
+if [ "$1" == "k8s_connectivity_check" ]; then
diff --git a/config/workers.coffee b/config/workers.coffee
@@ -116,13 +174,34 @@ module.exports = (KONFIG, options, credentials) ->
        ]
      healthCheckURLs   : [ "http://localhost:#{KONFIG.kloud.port}/healthCheck" ]
      versionURL        : "http://localhost:#{KONFIG.kloud.port}/version"
+      kubernetes        :
+        image           : 'koding/base'
diff --git a/config/generateRunFile.coffee b/config/generateRunFile.coffee

      export NAMESPACE_DIR="${KONFIG_PROJECTROOT}/deployment/kubernetes/namespace.yaml"
-      export BACKEND_DIR="${KONFIG_PROJECTROOT}/deployment/kubernetes/backend-pod/containers.yaml"
+      export EXT_SERVICES_DIR="${KONFIG_PROJECTROOT}/deployment/kubernetes/external-services/ -R"
diff --git a/.env-kubernetes b/.env-kubernetes
@@ -1,15 +1,22 @@
-export KONFIG_PROJECTROOT=/opt/koding
diff --git a/config/generateRunFile.coffee b/config/generateRunFile.coffee
-      envsubst < deployment/kubernetes/frontend-pod/client-containers.yaml.template > deployment/kubernetes/frontend-pod/client-containers.yaml
-
-      cp $BACKEND_DIR ${KONFIG_PROJECTROOT}/deployment/generated_files
+      cp $BACKEND_DIR ${KONFIG_PROJECTROOT}/deployment/generated_files -r
diff --git a/config/generateRunFile.coffee b/config/generateRunFile.coffee
-      kubectl exec -n $2 $1 -c $3 -- bash -c "./run is_ready" || exit 1
-    }
+      ${KONFIG_PROJECTROOT}/scripts/k8s-utilities.sh create_k8s_resource ${KONFIG_PROJECTROOT}/deployment/kubernetes/external-services/mongo
+      export MONGO_POD_NAME=$(kubectl get pods --namespace koding -l "app=mongo-ext-service" -o jsonpath="{.items[0].metadata.name}")
diff --git a/scripts/k8s-utilities.sh b/scripts/k8s-utilities.sh
+  remaining_time=600
+
+  until eval "check_pod_response $1 $2 $3"; do
+    sleep 30
diff --git a/config/generateRunFile.coffee b/config/generateRunFile.coffee
+      ${KONFIG_PROJECTROOT}/scripts/k8s-utilities.sh create_k8s_resource ${KONFIG_PROJECTROOT}/deployment/kubernetes/external-services/countly
+      export COUNTLY_POD_NAME=$(kubectl get pods --namespace koding -l "app=countly-ext-service" -o jsonpath="{.items[0].metadata.name}")
+      ${KONFIG_PROJECTROOT}/scripts/k8s-utilities.sh check_pod_state $COUNTLY_POD_NAME Pending
+      sleep 40
diff --git a/deployment/kubernetes.coffee b/deployment/kubernetes.coffee
+
+    KONFIG.kubernetesConf.spec.containers.push generateContainerSection name, workerOptions
+
+  KONFIG.kubernetesConf.spec.volumes = generateVolumesSection KONFIG, options
diff --git a/deployment/kubernetes/external-services/mongo/mongo-repctl.yaml b/deployment/kubernetes/external-services/mongo/mongo-repctl.yaml
@@ -0,0 +1,25 @@
+apiVersion: v1

		KONFIG.kubernetesConf.spec.containers.push generateContainerSection name, workerOptions

		KONFIG.kubernetesConf.spec.volumes = generateVolumesSection KONFIG, options