SDK
/
toaster.do
2
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

db storage, config fixes and some reformatting 

started using intellij cursive for some live debugging, which altered
the indentation of files
This commit is contained in:
Jaromil 2018-10-08 12:49:19 +02:00
parent a7deb67864
commit c277e7a6a8
6 changed files with 306 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
clojure_frontend/src/toaster/config.clj
View File

@ -93,7 +93,7 @@
;; (s/validate Config conf)
;; (catch Exception ex
;; (f/fail (log/spy :error ["Invalid configuration: " conf ex]))))
(get-in conf path))
(get-in conf (into [:toaster] path)))
(defn load-config [name default]
(log/info (str "Loading configuration: " name))

19
clojure_frontend/src/toaster/handler.clj
View File

@ -41,12 +41,12 @@
[toaster.ring :as ring]
[toaster.views :as views]
[toaster.jobs :as job])
(:import java.io.File)
(:gen-class))
(defonce config (conf/load-config "toaster" conf/default-settings))
(defroutes app-routes
(defroutes
app-routes
(GET "/" request (web/render "Hello World!")) ;; web/readme))
@ -60,6 +60,11 @@
(->> views/dockerfile-upload-post
(s/check request)))
(GET "/edit" request
(->> (fn [req conf acct]
(web/render acct views/dockerfile-edit-form))
(s/check request)))
(GET "/list" request
(->> views/list-jobs
(s/check request)))
@ -173,7 +178,9 @@
;; for uberjar
(defn -main []
(println "Starting standalone jetty server on http://localhost:6060")
(run-jetty app {:port 6060
:host "localhost"
:join? true}))
(println "Starting ring server")
(ring/init ring/app-defaults)
;(run-jetty app {:port 6060
; :host "localhost"
; :join? true})
)

55
clojure_frontend/src/toaster/jobs.clj
View File

@ -4,22 +4,58 @@
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-storage.db.mongo :refer [create-mongo-store]]
[clj-storage.core :as db]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.webpage :as web]
[toaster.config :refer :all]
[toaster.ring :refer [jobs]]
[hiccup.form :as hf]))
(defn- ssh-host [config]
(str (q config [:jenkins :user]) "@" (q config [:jenkins :host])))
(defn- sync_jobs [config arg1 arg2]
(with-programs
[ssh]
(try
(-> (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "sync_jobs.py"
arg1 arg2)
(str/split #"\n"))
(catch Exception e (f/fail (str "ERROR in sync_jobs.py - " (.getMessage e)))))))
(defn- dockerlint [path]
(with-programs [node]
(try
(node "node_modules/dockerlint/bin/dockerlint.js" path)
(catch Exception e
(f/fail (str "ERROR in dockerlint - " (.getMessage e)))))))
(defn add [path config account]
(with-programs [ssh scp node]
(let [tstamp (tc/to-long (time/now))
jobname (str (:email account) "-vm_amd64-" tstamp)
jobdir (str "/srv/toaster/" jobname)]
(f/attempt-all
[r_lint (node "node_modules/dockerlint/bin/dockerlint.js" path)
r_mkdir (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "mkdir" "-p" jobdir )
r_scp (scp "-i" "../id_ed25519" path (str "jenkins@sdk.bridge:" jobdir))
r_job (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-a" jobname)]
[r_lint (dockerlint path)
r_mkdir (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "mkdir" "-p" jobdir)
r_scp (scp "-i" (q config [:jenkins :key])
path (str (ssh-host config) ":" jobdir))
r_job (log/spy (sync_jobs config "-a" jobname))
r_store (log/spy (db/store!
@jobs :jobid
(log/spy {:jobid jobname
:email (:email account)
:account (dissoc account :password :activation-link)
:lint (if (.contains r_lint "is OK") true false)
:timestamp tstamp
:type "vm_amd64"})))]
{:lint r_lint
:job r_job}
(f/when-failed [e]
@ -28,13 +64,8 @@
(defn listall [config account]
(with-programs [ssh]
(f/attempt-all
[r_ssh (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-l" (:email account))]
(str/split r_ssh #"\n")
(f/when-failed [e]
(web/render-error-page
(str "Job list failure: " (f/message e)))))))
(sync_jobs config "-l" (:email account))
;;(db/query @jobs {:email (:email account)})
)

29
clojure_frontend/src/toaster/ring.clj
View File

@ -37,39 +37,41 @@
(def db (atom {}))
(def accts (atom {}))
(def auth (atom {}))
(def jobs (atom {}))
(defn init []
(log/merge-config! {:level :debug
(log/merge-config!
{:level :debug
;; #{:trace :debug :info :warn :error :fatal :report}
;; Control log filtering by
;; namespaces/patterns. Useful for turning off
;; logging in noisy libraries, etc.:
;; :ns-whitelist ["agiladmin.*" "just-auth.*"]
;; i.e: :ns-whitelist ["agiladmin.*" "just-auth.*"]
:ns-blacklist ["org.eclipse.jetty.*"
"org.mongodb.driver.cluster"]})
;; ------------------
;; load configuration
(reset! config (conf/load-config
(or (System/getenv "toaster_conf") "toaster")
conf/default-settings))
(let [justauth-conf (get-in @config [:toaster :just-auth])]
;; --------------------------------
;; initialize authentication stores
(let [justauth-conf (conf/q @config [:just-auth])]
;; connect database (TODO: take parameters from configuration)
(reset! db (get-mongo-db (:mongo-url justauth-conf)))
;; create authentication stores in db
(f/attempt-all
[auth-conf (get-in @config [:toaster :just-auth])
auth-stores (auth-db/create-auth-stores @db)]
[auth-stores (auth-db/create-auth-stores @db)]
[(trans/init "lang/auth-en.yml" "lang/english.yaml")
(reset! accts auth-stores)
(reset! auth (auth/email-based-authentication
auth-stores
;; TODO: replace with email taken from config
(dissoc (:just-auth (:toaster (conf/load-config
"toaster" conf/default-settings)))
(dissoc (conf/q @config [:just-auth])
:mongo-url :mongo-user :mongo-pass)
{:criteria #{:email :ip-address}
:type :block
@ -80,6 +82,13 @@
(f/when-failed [e]
(log/error (str (trans/locale [:init :failure])
" - " (f/message e))))))
;; ----------------------
;; initialize jobs stores
(reset! jobs (create-mongo-store @db :job-store))
;; ------------------------------
;; log all results worth noticing
(log/info (str (trans/locale [:init :success])))
(log/debug @auth))
@ -87,7 +96,7 @@
(-> site-defaults
(assoc-in [:cookies] true)
(assoc-in [:security :anti-forgery]
(get-in @config [:webserver :anti-forgery]))
(conf/q @config [:webserver :anti-forgery]))
(assoc-in [:security :ssl-redirect]
(get-in @config [:webserver :ssl-redirect]))
(conf/q @config [:webserver :ssl-redirect]))
(assoc-in [:security :hsts] true)))

36
clojure_frontend/src/toaster/views.clj
View File

@ -2,6 +2,7 @@
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.contrib.humanize :as humanize :refer [datetime]]
;; [clojure.data.json :as json :refer [read-str]]
[toaster.webpage :as web]
[toaster.session :as s]
@ -33,6 +34,25 @@ proceed to validation."]
:id "field-submit" :type "submit"
:name "submit" :value "submit"}]]]])
(def dockerfile-edit-form
[:div {:class "container-fluid"}
[:h1 "Edit your Dockerfile to toast"]
[:div {:class "form-group"}
[:form {:action "dockerfile" :method "post"
:class "form-shell"
:enctype "multipart/form-data"}
[:fieldset {:class "fieldset btn btn-default btn-file btn-lg"}
[:textarea {:name "editor" :id "editor"
:rows 30 :cols 72 } "FROM: dyne/devuan:ascii"]
[:input {:class "btn btn-primary btn-lg"
:id "field-submit" :type "submit"
:name "submit" :value "submit"}]]]]
[:script "var editor = ace.edit(\"editor\");
editor.setTheme(\"ace/theme/monokai\");
editor.session.setMode(\"ace/mode/dockerfile\");"]])
(def welcome-menu
[:div {:class "container-fluid"}
[:div {:class "row-fluid"}
@ -66,6 +86,8 @@ proceed to validation."]
(web/render-yaml (job/add path config account))]))))))
(defn list-jobs [request config account]
(f/attempt-all
[joblist (job/listall config account)]
(web/render
account
[:div {:class "container-fluid"}
@ -74,12 +96,18 @@ proceed to validation."]
[:thead nil
[:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]]
[:tbody nil
(for [j (job/listall config account)]
(for [j joblist]
(let [type (-> j (str/split #"-") second)
tstamp (-> j (str/split #"-") last)]
[:tr nil
[:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time)]
[:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/" j)} type]]
[:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time
humanize/datetime)]
[:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/"
(str/replace j #"@" "AT"))} type]]
[:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j))
(web/button "/remove" "Remove" (hf/hidden-field "job" j))]]
))]]]))
))]]])
(f/when-failed [e]
(web/render-error-page
(str "Job list failure: " (f/message e))))
))

6
clojure_frontend/toaster.yaml
View File

@ -2,8 +2,10 @@ webserver:
anti-forgery: false
ssl-redirect: false
admin:
email: jaromil@dyne.org
jenkins:
host: "bridge.toaster"
user: "jenkins"
key: "../id_ed25519"
just-auth:
email-server: "mail.dyne.org"