oly
/
kube
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updated deps

This commit is contained in:
Oly 2019-11-05 16:51:00 +00:00
parent aa4c393659
commit c8db5a253d
6 changed files with 255 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
deps.edn Normal file
View File

@ -0,0 +1,12 @@
{:deps
{org.clojure/clojure {:mvn/version "1.10.0"}
org.clojure/data.codec {:mvn/version "0.1.1"}
cheshire {:mvn/version "5.9.0"}
brosenan/lambdakube {:mvn/version "0.10.1-SNAPSHOT"}
org.flatland/ordered {:mvn/version "1.5.7"}
clj-commons/clj-yaml {:mvn/version "0.7.0"}
; io.forward/yaml {:mvn/version "1.0.9"}
nano-id {:mvn/version "0.10.0"}
cli-matic {:mvn/version "0.3.8"}}
:paths ["resources" "src"]}

2
project.clj
View File

@ -9,7 +9,7 @@
[cli-matic "0.3.8"]
[cheshire "5.9.0"]
[io.forward/yaml "1.0.9"]
[brosenan/lambdakube "0.7.0"]]
[brosenan/lambdakube "0.10.1-SNAPSHOT"]]
:plugins [[lein-auto "0.1.3"]]
:auto {:default {:file-pattern #"\.(clj|cljs|cljx|cljc|edn)$"}}
:main kube-deploy.core)

5
readme.org
View File

@ -1,3 +1,8 @@
# Auto deploy script
Based on (https://github.com/brosenan/lambdakube-example) using (https://github.com/brosenan/lambda-kube) library.
clj -m kube-deploy.core secrets --service="footprint-stage"
clj -m kube-deploy.core deploy --namespace="hackspace-stage"

164
src/kube_deploy/blocks.clj
View File

@ -38,8 +38,7 @@
(lk/expose-cluster-ip :mosquitto-master
(lk/port :mosquitto :mosquitto 1883 1883)))))))
(defn test [kube-injector]
(defn test-block [kube-injector]
(-> kube-injector
(lk/rule :test-rule []
(fn []
@ -53,14 +52,103 @@
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])))
(lk/deployment 1))))))
(defn postgres
(defn ldap
([kube-injector]
(postgres kube-injector {}))
(ldap kube-injector {}))
([kube-injector settings]
(let [app-name (get settings :app-name "ldap")
app-pod-name (str app-name "-pod")
app-rule-name (keyword (str app-name "-rule"))
app-container-name (keyword (str app-name "-container"))
app-service-name (keyword (str app-name "-service"))
app-labels (get settings :labels {:app "ldap"})
app-image (get settings :image "osixia/openldap:1.2.0")
app-resources (get settings :resources resources-small)]
(-> kube-injector
(lk/rule app-rule-name []
(fn []
(-> (lk/pod app-pod-name app-labels)
(lk/add-container
app-container-name
app-image
(-> {}
(lkh/add-to-spec :resources app-resources)))
(lk/deployment 1)
(lk/expose-cluster-ip
app-service-name
(lk/port app-container-name :ldap 389 389)))))))))
#_(defn aq-schedular
([kube-injector]
(aq-schedular kube-injector {}))
([kube-injector settings])
(let [app-name (get settings :app-name "aq-schedular")
app-pod-name (str app-name "-pod")
app-rule-name (keyword (str app-name "-rule"))
app-container-name (keyword (str app-name "-container"))
app-service-name (keyword (str app-name "-service"))
app-labels (get settings :labels {:app "aq-schedular"})
app-image (get settings :image "aq-schedular")
app-resources (get settings :resources resources-small)]
(lk/rule :footprint-aqscheduler-master []
(fn []
(-> (lk/pod :footprint-scheduler
{:app :footprint
:env :production
:tier :scheduler}
{:imagePullSecrets [{:name "docker-registry"}]})
(lk/add-container
:scheduler
app-image
(-> {}
(lkh/add-to-spec :imagePullPolicy "Always")
(lkh/add-to-spec :resources resources-mini)
(lkh/add-to-spec :command ["/usr/local/bin/dramatiq_apscheduler" "/app/tasks.yml" "--debug"])
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])))
(add-volume "scheduler-config" {:configMap
{:name "scheduler-tasks"}}
{:scheduler
{:name "scheduler-config"
:mountPath "/app/tasks.yml"
:subPath "tasks.yml"}})
(lk/deployment 1))))))
(defn redis
([kube-injector]
(redis kube-injector {}))
([kube-injector settings]
(let [app-name (get settings :app-name "redis")
app-pod-name (str app-name "-pod")
app-rule-name (keyword (str app-name "-rule"))
app-container-name (keyword (str app-name "-container"))
app-service-name (keyword (str app-name "-service"))
app-labels (get settings :labels {:app "redis"})
app-image (get settings :image "redis")
app-resources (get settings :resources resources-small)]
(-> kube-injector
(lk/rule app-rule-name []
(fn []
(-> (lk/pod app-pod-name app-labels)
(lk/add-container
app-container-name
app-image
(-> {}
(lkh/add-to-spec :resources app-resources)
(lkh/add-readiness-probe-cmd ["redis-cli" "--version"])
(lkh/add-liveness-probe-cmd ["redis-cli" "--version"])))
(lk/deployment 1)
(lk/expose-cluster-ip
app-service-name
(lk/port app-container-name :redis 6379 6379)))))))))
(defn rabbitmq
([kube-injector]
(rabbitmq kube-injector {}))
([kube-injector settings]
(let [app-name (get settings :app-name "postgres")
app-pod-name (str app-name "-pod")
app-rule-name (str app-name "-rule")
app-rule-name (keyword (str app-name "-rule"))
app-container-name (keyword (str app-name "-container"))
app-service-name (keyword (str app-name "-service"))
app-dependencies (get settings :dependencies [])
@ -69,7 +157,38 @@
app-image (get settings :image "postgres")
app-resources (get settings :resources resources-small)]
(-> kube-injector
(lk/rule :postgres-rule []
(lk/rule app-rule-name []
(fn []
(-> (lk/pod
app-pod-name
app-labels)
(lk/add-container
app-container-name
app-image
(-> {}
(lkh/add-to-spec :resources app-resources)
(lkh/add-liveness-probe-cmd ["rabbitmq-diagnostics" "ping" "-q"])))
(lk/deployment app-containers)
(lk/expose-cluster-ip
app-service-name
(lk/port app-container-name :rabbit 5672 5672)))))))))
(defn postgres
([kube-injector]
(postgres kube-injector {}))
([kube-injector settings]
(let [app-name (get settings :app-name "postgres")
app-pod-name (str app-name "-pod")
app-rule-name (keyword (str app-name "-rule"))
app-container-name (keyword (str app-name "-container"))
app-service-name (keyword (str app-name "-service"))
app-dependencies (get settings :dependencies [])
app-containers 1
app-labels (get settings :labels {:app "postgres"})
app-image (get settings :image "postgres")
app-resources (get settings :resources resources-small)]
(-> kube-injector
(lk/rule app-rule-name []
(fn []
(-> (lk/pod
app-pod-name
@ -81,12 +200,11 @@
(lkh/add-liveness-probe-cmd ["pg_isready"])
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])))
(lk/stateful-set 1)
(lk/add-volume-claim-template :postgres-volume
{:accessModes ["ReadWriteOnce"]
:resources {:requests {:storage "1Gi"}}}
{:bar "/var/lib/postgresql/data"})
(lk/expose-cluster-ip app-service-name ;app-name
(lk/expose-cluster-ip app-service-name
(lk/port app-container-name :postgres 5432 5432)))))))))
(defn gunicorn
@ -109,12 +227,9 @@
app-nginx-service-name "nginx-service"
app-serve-static (get settings :serve-static true)]
(-> kube-injector
(lk/rule (keyword (str app-name "-rule")) app-dependencies
(fn []
(println app-pod-name)
(as->
(lk/pod
app-pod-name
@ -152,14 +267,6 @@
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])
(lkh/add-to-spec :command ["./manage.py" "migrate"])))
#_(lk/add-init-container
pod
(keyword (str app-name "-init-static-files"))
app-image
(-> {}
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])
(lkh/add-to-spec :command ["./manage.py" "collectstatic" "--no-input"])))
(if (nil? app-serve-static)
pod
(-> pod
@ -172,6 +279,13 @@
;(lkh/add-to-spec :ports [{:containerPort 80 :name "nginx-port"}])
#_(lkh/add-readiness-probe-get "127.0.0.1/static/images/favicon.ico" 80)
(lkh/add-liveness-probe-get "http://127.0.0.1/static/admin/css/responsive.css" 80)))
(lk/add-init-container
(keyword (str app-name "-init-static-files"))
app-image
(-> {}
(lkh/add-to-spec :envFrom [{:secretRef {:name "environment"}}])
(lkh/add-to-spec :command ["./manage.py" "collectstatic" "--no-input"])))
(lk/add-volume :static-files
{:emptyDir {}}
{app-nginx-container-name "/usr/share/nginx/html/static/"
@ -182,8 +296,9 @@
;(lku/wait-for-service-port postgres :postgres)
;(lku/wait-for-service-port redis :redis)
; wrap in a deployment
(lk/deployment pod 1)
(if (nil? app-serve-static)
pod
@ -195,10 +310,11 @@
(lk/expose-cluster-ip
pod
app-service-name
(lk/port app-gunicorn-container-name :gunicorn-port 5000 5000))
)))
(lk/port app-gunicorn-container-name :gunicorn-port 5000 5000)))))
; inject the ingress required to serve to gunicorn app
(lk/rule (keyword (str app-name "-ingress-rule")) []
(fn []
(let [ingress-name (str app-name "-ingress")
@ -222,7 +338,6 @@
; (lk/add-annotation :certmanager.k8s.io/cluster-issuer "letsencrypt-production")
(lk/add-annotation :kubernetes.io/ingress.class "nginx")))))))))
(defn django-huey
([kube-injector]
(gunicorn kube-injector {}))
@ -237,7 +352,6 @@
app-image (get settings :image "python:3")
app-resources (get settings :resources resources-small)]
(-> kube-injector
(lk/rule app-rule-name []
(fn []

121
src/kube_deploy/core.clj
View File

@ -41,11 +41,11 @@
(map :metadata)
(map :name))))
(defn kube-does-namespace-exists [namespace]
(some namespace
(into #{} (->> (lkh/kube-namespaces "/home/oly/.kube/dke.yml")
(map :metadata)
(map :name)))))
(defn kube-does-namespace-exists
([namespace]
(kube-does-namespace-exists namespace nil))
([namespace kube-config]
(some #(= (name namespace) %) (kube-get-namespaces kube-config))))
@ -381,7 +381,7 @@
(fn []
(lk/config-map name config))))
(defn make-ingress-hackspace [kube-injector]
#_(defn make-ingress-hackspace [kube-injector]
(lk/rule kube-injector :lkube-ingress []
(fn []
(-> (lkh/ingress "lkube-ingress" {:app :nginx-ingress :env :production :tier :ingress})
@ -399,7 +399,7 @@
(lk/add-annotation :certmanager.k8s.io/cluster-issuer "letsencrypt-production")
(lk/add-annotation :kubernetes.io/ingress.class "nginx")))))
(defn make-ingress [kube-injector]
#_(defn make-ingress [kube-injector]
(lk/rule kube-injector :lkube-ingress []
(fn []
(-> (lkh/ingress "lkube-ingress" {:app :nginx-ingress :env :production :tier :ingress})
@ -417,9 +417,8 @@
(lk/add-annotation :kubernetes.io/tls-acme "true")
(lk/add-annotation :kubernetes.io/ingress.class "nginx")))))
(defn footprint-deployment [namespace kube-config]
#_(defn footprint-deployment [namespace kube-config]
(println "deploying footprint")
(reset! conf (clojure.edn/read-string (slurp (str namespace "-config.edn"))))
; run deployment in default namespace to create a namespace called footprint
(-> (lk/injector)
@ -427,7 +426,7 @@
lk/standard-descs
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-store.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) nil kube-config))
(lk/kube-apply (io/file (str namespace "-deploy.yaml")) nil kube-config))
; run deployment under footprint namespace
(-> (lk/injector)
@ -442,87 +441,68 @@
lk/standard-descs
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-store.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) "footprint" kube-config)))
(defn hackspace-deployment [namespace kube-config]
(println "deploying hackspace")
(reset! conf (clojure.edn/read-string (slurp (str namespace "-config.edn"))))
; run deployment in default namespace to create a namespace called footprint
(-> (lk/injector)
(make-namespace namespace)
;lk/standard-descs
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-store.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) nil kube-config))
; run deployment under footprint namespace
(-> (lk/injector)
(make-environment-secrets :environment (:vars @conf))
;(make-registry-secrets :docker-registry {:.dockerconfigjson (:registry @conf)})
make-ingress-hackspace
database
redis
hackspace
lk/standard-descs
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-store.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) namespace kube-config)))
;(map lk/injector 'database)
(lk/kube-apply (io/file (str namespace "-deploy.yaml")) "footprint" kube-config)))
(defn create-namespace-if-not-exists [{:keys [config namespace blocks]}]
(when-not (kube-does-namespace-exists (keyword namespace))
(when-not (kube-does-namespace-exists (keyword namespace) config)
(println (str "Namespace " namespace " not found creating"))
(-> (lk/injector)
(make-namespace namespace)
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-namespace.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) nil config))))
(lk/kube-apply (io/file (str namespace "-deploy.yaml")) nil config))))
; recursive function that processes through each project block
(defn apply-blocks [kube-injector func settings]
(println (first func))
(println kube-injector)
(if (= 0 (count func))
kube-injector
(if (some? (first func))
(do
(println "Generating block" (first func))
(recur
(case (first func)
:database (database kube-injector)
:ldap (block/ldap kube-injector)
:redis (block/redis kube-injector)
:rabbitmq (block/rabbitmq kube-injector)
:postgres (block/postgres kube-injector)
:mosquitto (block/mosquitto-mqtt kube-injector)
:django-huey (block/django-huey kube-injector (get-in settings [(keyword (first func))]))
:django-gunicorn (block/gunicorn kube-injector (get-in settings [(keyword (first func))]))
kube-injector)
(rest func)
settings)))
settings))
kube-injector))
(defn build [{:keys [config namespace blocks] :as service}]
(println "Deploying" config namespace)
(println (str namespace "-store.yaml"))
; build functionality, pass config to apply blocks to generate the map then dump yaml
(defn build [{:keys [config namespace blocks options] :as service}]
(println "Deploying to" (or config "default") "namespace" namespace)
(println options)
(if (.exists (io/file (str namespace "-config.edn")))
(do (reset! conf (clojure.edn/read-string (slurp (str namespace "-config.edn"))))
(create-namespace-if-not-exists service)
(let [injector (lk/injector)]
(-> injector
(make-environment-secrets :environment (:vars @conf))
;(make-registry-secrets :docker-registry {:.dockerconfigjson (:registry @conf)})
;(apply-blocks [:postgres] @conf)
;(apply-blocks [:django-gunicorn] @conf)
(apply-blocks blocks @conf)
lk/standard-descs
(lk/get-deployable deployment-config)
(lkh/to-yaml-store (str namespace "-store.yaml"))
(kube-apply (io/file (str namespace "-deploy.yaml")) namespace config))))
(as-> (lk/injector)
injector
(if (= "t" (:secrets options))
(make-environment-secrets injector :environment (:vars @conf))
injector)
;(make-registry-secrets injector :docker-registry {:.dockerconfigjson (:registry @conf)})
(apply-blocks injector blocks @conf)
(lk/standard-descs injector)
(lk/get-deployable injector deployment-config)
(lkh/to-yaml-store injector (str namespace "-store.yaml"))
(if (= "t" (:deploy options))
(lk/kube-apply injector (io/file (str namespace "-deploy.yaml")) namespace config))))
(println "No config found " (str namespace "-config.yaml"))))
; actual deploy config
(defn deploy [{:keys [namespace] :as ns-config}]
(println ns-config)
(let [config (get-in data/services [(keyword namespace) :config])
blocks (get-in data/services [(keyword namespace) :blocks])]
(cond
(= namespace "hackspace") (hackspace-deployment "maidstone-hackspace" "/home/oly/.kube/dke.yml")
(= namespace "footprint-production") (footprint-deployment "footprint-production" "/home/oly/.kube/ake.yml")
(= namespace "footprint-stage") (footprint-deployment "footprint" "/home/oly/.kube/ake.yml")
(= (nil? blocks)) (build {:namespace namespace :config config :blocks blocks})
;(= namespace "footprint-production") (footprint-deployment "footprint-production" "/home/oly/.kube/ake.yml")
;(= namespace "footprint-stage") (footprint-deployment "footprint" "/home/oly/.kube/ake.yml")
(= (nil? blocks)) (build {:namespace namespace :config config :blocks blocks :options ns-config})
:else (println (str "Namespace " namespace " not found.")))))
(def CONFIGURATION
@ -532,7 +512,9 @@
; :global-opts []
:commands [{:command "deploy"
:description "Deploy app out to cluster"
:opts [{:option "namespace" :as "Namespace to deploy" :type :string :spec string?}]
:opts [{:short "n" :option "namespace" :as "Namespace to deploy" :type :string :spec string?}
{:short "d" :option "deploy" :as "Deploy with kubectl" :type :string :default nil}
{:short "s" :option "secrets" :as "Include secrets" :type :string :default nil}]
:runs deploy}
{:command "secrets"
:description "Display secrests"
@ -540,16 +522,7 @@
:runs lky/fetch-secrets}]})
(defn -main [& args]
;(deploy "hackspace")
(if args
(run-cmd args CONFIGURATION)
(do
#_(hackspace-deployment "maidstone-hackspace" "/home/oly/.kube/dke.yml")
(footprint-deployment "footprint" "/home/oly/.kube/ake.yml")))
#_(hackspace-deployment "maidstone-hackspace" "/home/oly/.kube/dke.yml")
#_(footprint-deployment "footprint-production" "/home/oly/.kube/ake.yml")
#_(footprint-deployment "footprint" "/home/oly/.kube/ake.yml"))
(run-cmd args CONFIGURATION)))
;/usr/local/bin/gunicorn config.wsgi --workers 2 -k gevent --worker-connections 100 --max-requests 300 --keep-alive 1000 -b 0.0.0.0:5000 --chdir=/app

1
src/kube_deploy/data.clj
View File

@ -1,4 +1,3 @@
(ns kube-deploy.data)
(def services (clojure.edn/read-string (slurp "config.edn")))
(def fragments (clojure.edn/read-string (slurp "fragments.edn")))