From 30fb7ae1a253fe84e7dc5b9b73b1db1e014e9890 Mon Sep 17 00:00:00 2001 From: Jaromil Date: Sun, 14 Oct 2018 19:13:47 +0200 Subject: [PATCH] added profiles, for joblimit and future features also fixes to file upload, parameter reading and error handling --- .../resources/templates/body_addjob.html | 6 +- clojure_frontend/src/toaster/handler.clj | 18 ++- clojure_frontend/src/toaster/jobs.clj | 121 ++++++++++-------- clojure_frontend/src/toaster/profiles.clj | 36 ++++++ clojure_frontend/src/toaster/ring.clj | 7 +- clojure_frontend/src/toaster/session.clj | 46 +++++-- clojure_frontend/src/toaster/views.clj | 32 ++--- 7 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 clojure_frontend/src/toaster/profiles.clj diff --git a/clojure_frontend/resources/templates/body_addjob.html b/clojure_frontend/resources/templates/body_addjob.html index 8f03d25..d5944d6 100644 --- a/clojure_frontend/resources/templates/body_addjob.html +++ b/clojure_frontend/resources/templates/body_addjob.html @@ -1,10 +1,10 @@

Upload a Dockerfile to toast

Choose the file in your computer and click 'Submit' to proceed to validation.

-
-
+ +
diff --git a/clojure_frontend/src/toaster/handler.clj b/clojure_frontend/src/toaster/handler.clj index 769e364..ade1901 100644 --- a/clojure_frontend/src/toaster/handler.clj +++ b/clojure_frontend/src/toaster/handler.clj @@ -38,7 +38,8 @@ [toaster.session :as s] [toaster.config :as conf] [toaster.ring :as ring] - [toaster.views :as views]) + [toaster.views :as views] + [toaster.profiles :as profile]) (:gen-class)) (defonce config (conf/load-config "toaster" conf/default-settings)) @@ -81,9 +82,9 @@ (POST "/dockerfile" request (->> (fn [req conf acct] (s/render acct - [:body - (views/dockerfile-upload-post req conf acct) - (views/dashboard acct)])) + [:body + (views/dockerfile-upload req conf acct) + (views/dashboard acct)])) (auth-wrap request))) (POST "/remove" request @@ -115,7 +116,7 @@ (auth-wrap request))) ;; JUST-AUTH ROUTES - (GET "/login" request (s/render (s/resource s/login))) + (GET "/login" request (s/render-template s/login)) (POST "/login" request (f/attempt-all @@ -142,7 +143,7 @@ (s/render [:body [:h1 {:class "title"} "Logged out."]]))) - (GET "/signup" request (s/render (s/resource s/signup))) + (GET "/signup" request (s/render-template s/signup)) (POST "/signup" request (f/attempt-all [name (s/param request :name) @@ -194,7 +195,10 @@ [:h1 {:class "title"} "Failure activating account"] [:h2 {:class "subtitle"} (f/message act)] [:p (str "Email: " email " activation-id: " activation-id)]] "is-error") - (s/notify [:h1 {:class "title"} (str "Account activated - " email)] "is-success"))]))) + [:span + (s/notify (str "Account activated: " email) "is-success") + (profile/create email)] + )]))) ;; -- end of JUST-AUTH (POST "/" request diff --git a/clojure_frontend/src/toaster/jobs.clj b/clojure_frontend/src/toaster/jobs.clj index 650b2f3..df66216 100644 --- a/clojure_frontend/src/toaster/jobs.clj +++ b/clojure_frontend/src/toaster/jobs.clj @@ -1,33 +1,33 @@ (ns toaster.jobs (:require - [clojure.string :as str] - [clojure.java.io :as io] - [clj-time.core :as time] - [clj-time.coerce :as tc] - [clj-storage.db.mongo :refer [create-mongo-store]] - [clj-storage.core :as db] - [failjure.core :as f] - [taoensso.timbre :as log :refer [debug]] - [me.raynes.conch :as sh :refer [with-programs]] - [toaster.config :refer :all] - [toaster.ring :refer [jobs]] - [hiccup.form :as hf])) + [clojure.string :as str] + [clojure.java.io :as io] + [clj-time.core :as time] + [clj-time.coerce :as tc] + [clj-storage.core :as db] + [failjure.core :as f] + [taoensso.timbre :as log :refer [debug]] + [me.raynes.conch :as sh :refer [with-programs]] + [toaster.config :refer :all] + [toaster.ring :refer [jobs]] + [toaster.profiles :as profile] + [hiccup.form :as hf])) ;; list of arm targets to chose from -;"beagleboneblack" -;"chromeacer" -;"chromeveyron" -;"droid4" -;"n900" -;"odroidxu4" -;"odroidxu" -;"ouya" -;"raspi1" -;"raspi2" -;"raspi3" -;"rock64" -;"sunxi" -;"turbox-twister" + ;"beagleboneblack" + ;"chromeacer" + ;"chromeveyron" + ;"droid4" + ;"n900" + ;"odroidxu4" + ;"odroidxu" + ;"ouya" + ;"raspi1" + ;"raspi2" + ;"raspi3" + ;"rock64" + ;"sunxi" + ;"turbox-twister" (defn- ssh-host [config] (str (q config [:jenkins :user]) "@" (q config [:jenkins :host]))) @@ -46,36 +46,45 @@ (defn- dockerlint [path] (with-programs [node] - (try - (node "node_modules/dockerlint/bin/dockerlint.js" path) - (catch Exception e - (f/fail (str "ERROR in dockerlint - " (.getMessage e))))))) + (try + (node "node_modules/dockerlint/bin/dockerlint.js" path) + (catch Exception e + (f/fail (str "ERROR in dockerlint - " (.getMessage e))))))) +(defn count [id] + (f/attempt-all + [joblist (db/query @jobs {:email id})] + (clojure.core/count joblist) + (f/when-failed [e] + (f/fail "jobs/count :: " (f/message e))))) (defn add [path config account] - (with-programs [ssh scp node] - (let [tstamp (tc/to-long (time/now)) - jobname (str (:email account) "-vm_amd64_ascii-" tstamp) - jobdir (str "/srv/toaster/" jobname)] - (f/attempt-all - [r_lint (dockerlint path) - r_mkdir (ssh "-i" (q config [:jenkins :key]) - (ssh-host config) "mkdir" "-p" jobdir) - r_scp (scp "-i" (q config [:jenkins :key]) - path (str (ssh-host config) ":" jobdir "/Dockerfile")) - r_job (sync_jobs config "-a" jobname) - r_store (db/store! @jobs :jobid - {:jobid jobname - :email (:email account) - :account (dissoc account :password :activation-link) - :lint (if (.contains r_lint "is OK") true false) - :timestamp tstamp - :type "vm_amd64_ascii" - :dockerfile (slurp path)})] - {:lint r_lint - :job r_job} - (f/when-failed [e] - (f/fail (str "Job add '" jobname "' failure: " (f/message e)))))))) - - - + (with-programs + [ssh scp node] + (let [tstamp (tc/to-long (time/now)) + jobname (str (:email account) "-vm_amd64_ascii-" tstamp) + jobdir (str "/srv/toaster/" jobname)] + (f/if-let-ok? [joblimit (profile/get-joblimit (:email account))] + (if (>= (count (:email account)) joblimit) + (f/fail "Job limit is reached, trash some to free slots") + (f/attempt-all + [r_lint (dockerlint path) + r_mkdir (ssh "-i" (q config [:jenkins :key]) + (ssh-host config) "mkdir" "-p" jobdir) + r_scp (scp "-i" (q config [:jenkins :key]) + path (str (ssh-host config) ":" jobdir "/Dockerfile")) + r_job (sync_jobs config "-a" jobname) + r_store (db/store! @jobs :jobid + {:jobid jobname + :email (:email account) + :account (dissoc account :password :activation-link) + :lint (if (.contains r_lint "is OK") true false) + :timestamp tstamp + :type "vm_amd64_ascii" + :dockerfile (slurp path)})] + {:lint r_lint + :job r_job} + (f/when-failed [e] + (f/fail (str "Job add '" jobname "' failure :: " (f/message e)))))) + (f/fail (str "Cannot find profile " (:email account) " :: " + (f/message joblimit))))))) diff --git a/clojure_frontend/src/toaster/profiles.clj b/clojure_frontend/src/toaster/profiles.clj new file mode 100644 index 0000000..db4733e --- /dev/null +++ b/clojure_frontend/src/toaster/profiles.clj @@ -0,0 +1,36 @@ +(ns toaster.profiles + (:require + [clojure.string :as str] + [clojure.java.io :as io] + [clj-time.core :as time] + [clj-time.coerce :as tc] + [clj-storage.db.mongo :refer [create-mongo-store]] + [clj-storage.core :as db] + [failjure.core :as f] + [taoensso.timbre :as log :refer [debug]] + [me.raynes.conch :as sh :refer [with-programs]] + [toaster.config :refer :all] + [toaster.ring :refer [jobs profiles]] + [toaster.bulma :refer [render-yaml]] + [hiccup.form :as hf])) + +(defn create [id] + (let [profile {:email id + :joblimit 3 ; default + :roles ["noob"] + :lastlogin (tc/to-long (time/now))}] + (db/store! @profiles :email profile) + (render-yaml profile))) + +(defn- fetch-profile [id] + (f/if-let-ok? [profile (db/fetch @profiles id)] + profile + (f/fail (str "Profile not found " id " :: " (f/message profile))))) + +(defn get-joblimit [id] + (f/ok-> id fetch-profile (get :joblimit))) + +(defn has-role [id role] + (f/ok-> id fetch-profile (get :roles) + clojure.core/set (contains? role))) + diff --git a/clojure_frontend/src/toaster/ring.clj b/clojure_frontend/src/toaster/ring.clj index 14d4fe2..40c13a8 100644 --- a/clojure_frontend/src/toaster/ring.clj +++ b/clojure_frontend/src/toaster/ring.clj @@ -33,11 +33,14 @@ [ring.middleware.accept :refer [wrap-accept]] [ring.middleware.defaults :refer [wrap-defaults site-defaults]])) +;; generic webapp stores (def config (atom {})) (def db (atom {})) (def accts (atom {})) (def auth (atom {})) -(def jobs (atom {})) +;; app specific stores +(def jobs (atom {})) +(def profiles (atom {})) (defn init [] (log/merge-config! @@ -86,6 +89,8 @@ ;; ---------------------- ;; initialize jobs stores (reset! jobs (create-mongo-store @db :job-store)) + ;; initialize profile stores + (reset! profiles (create-mongo-store @db :profile-store)) ;; ------------------------------ ;; log all results worth noticing diff --git a/clojure_frontend/src/toaster/session.clj b/clojure_frontend/src/toaster/session.clj index 28c5855..2c06801 100644 --- a/clojure_frontend/src/toaster/session.clj +++ b/clojure_frontend/src/toaster/session.clj @@ -14,17 +14,18 @@ ([template] (resource template {})) ([template params] (render-resource template params))) -(defonce login "templates/body_loginform.html") +(defonce login "templates/body_loginform.html") (defonce signup "templates/body_signupform.html") -(defonce head "templates/html_head.html") +(defonce head "templates/html_head.html") (defonce footer "templates/body_footer.html") (defn param [request param] - (let [value - (get-in request - (conj [:params] param))] + (let [p (if (coll? param) + (into [:params] param) + (conj [:params] param)) + value (get-in request p)] (if (nil? value) - (f/fail (str "Parameter not found: " param)) + (f/fail (str "Parameter not found: " p)) value))) ;; TODO: not working? @@ -70,9 +71,12 @@ (= tclass "is-warning") (log/warn msg) (= tclass "is-success") (log/info msg) (= tclass "is-primary") (log/debug msg)) - [:div {:class (str "notification " tclass " has-text-centered")} msg]))) + [:div {:class (str "notification " tclass " has-text-centered")} msg] + ))) ;; shortcut -(defn error [msg fail] (notify (str msg " :: " (f/message fail)) "is-danger")) +(defn error + ([msg] (notify msg "is-danger")) + ([msg fail] (notify (str msg " :: " (f/message fail)) "is-danger"))) (defn render "render a full webpage using headers, navbar, body and footer" @@ -84,4 +88,30 @@ (resource head) (conj body (resource footer)))})) +(defn render-template + "render an html from resources using headers, navbar, body and footer" + ([body] (render-template nil body)) + ([account body] + {:headers {"Content-Type" + "text/html; charset=utf-8"} + :body (str "\n" + (resource head) + "\n\n" + (resource body) + "\n\n" + (resource footer))})) + +(defn render-html + "render an html from resources using headers, navbar, body and footer" + ([body] (render-template nil body)) + ([account body] + {:headers {"Content-Type" + "text/html; charset=utf-8"} + :body (str "\n" + (resource head) + "\n\n" + body ;; html text string + "\n\n" + (resource footer))})) + (defn render-error [err] (->> "is-danger" (notify err) render)) diff --git a/clojure_frontend/src/toaster/views.clj b/clojure_frontend/src/toaster/views.clj index d804613..c278b60 100644 --- a/clojure_frontend/src/toaster/views.clj +++ b/clojure_frontend/src/toaster/views.clj @@ -60,29 +60,31 @@ (web/button "/remove" "\uD83D\uDDD1" (hf/hidden-field "jobid" jobid))]]] ))]]]) -(defn dockerfile-upload-post [request config account] +(defn dockerfile-upload [request config account] (f/attempt-all [tempfile (s/param request [:file :tempfile]) filename (s/param request [:file :filename]) - filesize (s/param request [:file :size])] - (cond - (> filesize 64000) + params (log/spy (:params request))] + (if (> (s/param request [:file :size]) 64000) ;; TODO: put filesize limit in config - (s/notify "File too big in upload (64KB limit)" "is-danger") - :else + (s/error "File too big in upload (64KB limit)") + ;; else (let [file (io/copy tempfile (io/file "/tmp" filename)) path (str "/tmp/" filename)] (io/delete-file tempfile) (if (not (.exists (io/file path))) - (s/notify (str "Uploaded file not found: " filename) "is-danger") + (s/error (str "Uploaded file not found: " filename)) ;; file is now in 'tmp' var - (f/if-let-ok? [newjob (job/add path config account)] - [:div {:class "container"} - [:h1 {:class "title"} "Job uploaded and added"] - [:p "Log messages:"] - (web/render-yaml newjob)] - ;; else when job/add is not-ok - (s/error "Error adding job" newjob))))) + (f/attempt-all + [newjob (job/add path config account)] + [:div {:class "container"} + [:h1 {:class "title"} "Job uploaded and added"] + [:p "Log messages:"] + (web/render-yaml newjob)] + ;; else when job/add is not-ok + (f/when-failed [e] + (s/error "Error adding job" e)) + )))) (f/when-failed [e] (s/error "Upload file error" e)))) @@ -107,7 +109,7 @@ jobfound (db/query @ring/jobs {:jobid jobid}) r_rmjob (db/delete! @ring/jobs jobid) r_sync (job/sync_jobs config "-d" jobid)] - (s/notify (str "Job removed: " jobid) "is-primary") + (s/notify (str "Job removed :: " jobid) "is-primary") (f/when-failed [e] (s/error "Failure removing job" e))))