SDK
/
toaster.do
2
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

added profiles, for joblimit and future features 

also fixes to file upload, parameter reading and error handling
This commit is contained in:
Jaromil 2018-10-14 19:13:47 +02:00
parent f28dcb2888
commit 30fb7ae1a2
7 changed files with 176 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clojure_frontend/resources/templates/body_addjob.html
View File

@ -1,10 +1,10 @@
<div class="box">
<h1 class="title">Upload a Dockerfile to toast</h1>
<p>Choose the file in your computer and click 'Submit' to proceed to validation.</p>
<form action="dockerfile" class="form-shell" enctype="multipart/form-data" method="post">
<div class="file has-label is-fullwidth">
<form action="dockerfile" class="form" enctype="multipart/form-data" method="post">
<div class="has-label is-fullwidth">
<label class="file-label">
<input class="file-input inputfile inputfile-2" id="file" type="file" />
<input name="file" class="file-input inputfile inputfile-2" id="file" type="file" />
<label for="file"><span id="filename">Choose a Dockerfile...</span></label>
<span class="file-cta"><span class="file-icon"><i class="fa fa-upload"></i></span><span class="file-label">Upload</span></span>
</label></div>

18
clojure_frontend/src/toaster/handler.clj
View File

@ -38,7 +38,8 @@
[toaster.session :as s]
[toaster.config :as conf]
[toaster.ring :as ring]
[toaster.views :as views])
[toaster.views :as views]
[toaster.profiles :as profile])
(:gen-class))
(defonce config (conf/load-config "toaster" conf/default-settings))
@ -81,9 +82,9 @@
(POST "/dockerfile" request
(->> (fn [req conf acct]
(s/render acct
[:body
(views/dockerfile-upload-post req conf acct)
(views/dashboard acct)]))
[:body
(views/dockerfile-upload req conf acct)
(views/dashboard acct)]))
(auth-wrap request)))
(POST "/remove" request
@ -115,7 +116,7 @@
(auth-wrap request)))
;; JUST-AUTH ROUTES
(GET "/login" request (s/render (s/resource s/login)))
(GET "/login" request (s/render-template s/login))
(POST "/login" request
(f/attempt-all
@ -142,7 +143,7 @@
(s/render [:body
[:h1 {:class "title"} "Logged out."]])))
(GET "/signup" request (s/render (s/resource s/signup)))
(GET "/signup" request (s/render-template s/signup))
(POST "/signup" request
(f/attempt-all
[name (s/param request :name)
@ -194,7 +195,10 @@
[:h1 {:class "title"} "Failure activating account"]
[:h2 {:class "subtitle"} (f/message act)]
[:p (str "Email: " email " activation-id: " activation-id)]] "is-error")
(s/notify [:h1 {:class "title"} (str "Account activated - " email)] "is-success"))])))
[:span
(s/notify (str "Account activated: " email) "is-success")
(profile/create email)]
)])))
;; -- end of JUST-AUTH
(POST "/" request

121
clojure_frontend/src/toaster/jobs.clj
View File

@ -1,33 +1,33 @@
(ns toaster.jobs
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-storage.db.mongo :refer [create-mongo-store]]
[clj-storage.core :as db]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.config :refer :all]
[toaster.ring :refer [jobs]]
[hiccup.form :as hf]))
[clojure.string :as str]
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-storage.core :as db]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.config :refer :all]
[toaster.ring :refer [jobs]]
[toaster.profiles :as profile]
[hiccup.form :as hf]))
;; list of arm targets to chose from
;"beagleboneblack"
;"chromeacer"
;"chromeveyron"
;"droid4"
;"n900"
;"odroidxu4"
;"odroidxu"
;"ouya"
;"raspi1"
;"raspi2"
;"raspi3"
;"rock64"
;"sunxi"
;"turbox-twister"
;"beagleboneblack"
;"chromeacer"
;"chromeveyron"
;"droid4"
;"n900"
;"odroidxu4"
;"odroidxu"
;"ouya"
;"raspi1"
;"raspi2"
;"raspi3"
;"rock64"
;"sunxi"
;"turbox-twister"
(defn- ssh-host [config]
(str (q config [:jenkins :user]) "@" (q config [:jenkins :host])))
@ -46,36 +46,45 @@
(defn- dockerlint [path]
(with-programs [node]
(try
(node "node_modules/dockerlint/bin/dockerlint.js" path)
(catch Exception e
(f/fail (str "ERROR in dockerlint - " (.getMessage e)))))))
(try
(node "node_modules/dockerlint/bin/dockerlint.js" path)
(catch Exception e
(f/fail (str "ERROR in dockerlint - " (.getMessage e)))))))
(defn count [id]
(f/attempt-all
[joblist (db/query @jobs {:email id})]
(clojure.core/count joblist)
(f/when-failed [e]
(f/fail "jobs/count :: " (f/message e)))))
(defn add [path config account]
(with-programs [ssh scp node]
(let [tstamp (tc/to-long (time/now))
jobname (str (:email account) "-vm_amd64_ascii-" tstamp)
jobdir (str "/srv/toaster/" jobname)]
(f/attempt-all
[r_lint (dockerlint path)
r_mkdir (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "mkdir" "-p" jobdir)
r_scp (scp "-i" (q config [:jenkins :key])
path (str (ssh-host config) ":" jobdir "/Dockerfile"))
r_job (sync_jobs config "-a" jobname)
r_store (db/store! @jobs :jobid
{:jobid jobname
:email (:email account)
:account (dissoc account :password :activation-link)
:lint (if (.contains r_lint "is OK") true false)
:timestamp tstamp
:type "vm_amd64_ascii"
:dockerfile (slurp path)})]
{:lint r_lint
:job r_job}
(f/when-failed [e]
(f/fail (str "Job add '" jobname "' failure: " (f/message e))))))))
(with-programs
[ssh scp node]
(let [tstamp (tc/to-long (time/now))
jobname (str (:email account) "-vm_amd64_ascii-" tstamp)
jobdir (str "/srv/toaster/" jobname)]
(f/if-let-ok? [joblimit (profile/get-joblimit (:email account))]
(if (>= (count (:email account)) joblimit)
(f/fail "Job limit is reached, trash some to free slots")
(f/attempt-all
[r_lint (dockerlint path)
r_mkdir (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "mkdir" "-p" jobdir)
r_scp (scp "-i" (q config [:jenkins :key])
path (str (ssh-host config) ":" jobdir "/Dockerfile"))
r_job (sync_jobs config "-a" jobname)
r_store (db/store! @jobs :jobid
{:jobid jobname
:email (:email account)
:account (dissoc account :password :activation-link)
:lint (if (.contains r_lint "is OK") true false)
:timestamp tstamp
:type "vm_amd64_ascii"
:dockerfile (slurp path)})]
{:lint r_lint
:job r_job}
(f/when-failed [e]
(f/fail (str "Job add '" jobname "' failure :: " (f/message e))))))
(f/fail (str "Cannot find profile " (:email account) " :: "
(f/message joblimit)))))))

36
clojure_frontend/src/toaster/profiles.clj Normal file
View File

@ -0,0 +1,36 @@
(ns toaster.profiles
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-storage.db.mongo :refer [create-mongo-store]]
[clj-storage.core :as db]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.config :refer :all]
[toaster.ring :refer [jobs profiles]]
[toaster.bulma :refer [render-yaml]]
[hiccup.form :as hf]))
(defn create [id]
(let [profile {:email id
:joblimit 3 ; default
:roles ["noob"]
:lastlogin (tc/to-long (time/now))}]
(db/store! @profiles :email profile)
(render-yaml profile)))
(defn- fetch-profile [id]
(f/if-let-ok? [profile (db/fetch @profiles id)]
profile
(f/fail (str "Profile not found " id " :: " (f/message profile)))))
(defn get-joblimit [id]
(f/ok-> id fetch-profile (get :joblimit)))
(defn has-role [id role]
(f/ok-> id fetch-profile (get :roles)
clojure.core/set (contains? role)))

7
clojure_frontend/src/toaster/ring.clj
View File

@ -33,11 +33,14 @@
[ring.middleware.accept :refer [wrap-accept]]
[ring.middleware.defaults :refer [wrap-defaults site-defaults]]))
;; generic webapp stores
(def config (atom {}))
(def db (atom {}))
(def accts (atom {}))
(def auth (atom {}))
(def jobs (atom {}))
;; app specific stores
(def jobs (atom {}))
(def profiles (atom {}))
(defn init []
(log/merge-config!
@ -86,6 +89,8 @@
;; ----------------------
;; initialize jobs stores
(reset! jobs (create-mongo-store @db :job-store))
;; initialize profile stores
(reset! profiles (create-mongo-store @db :profile-store))
;; ------------------------------
;; log all results worth noticing

46
clojure_frontend/src/toaster/session.clj
View File

@ -14,17 +14,18 @@
([template] (resource template {}))
([template params] (render-resource template params)))
(defonce login "templates/body_loginform.html")
(defonce login "templates/body_loginform.html")
(defonce signup "templates/body_signupform.html")
(defonce head "templates/html_head.html")
(defonce head "templates/html_head.html")
(defonce footer "templates/body_footer.html")
(defn param [request param]
(let [value
(get-in request
(conj [:params] param))]
(let [p (if (coll? param)
(into [:params] param)
(conj [:params] param))
value (get-in request p)]
(if (nil? value)
(f/fail (str "Parameter not found: " param))
(f/fail (str "Parameter not found: " p))
value)))
;; TODO: not working?
@ -70,9 +71,12 @@
(= tclass "is-warning") (log/warn msg)
(= tclass "is-success") (log/info msg)
(= tclass "is-primary") (log/debug msg))
[:div {:class (str "notification " tclass " has-text-centered")} msg])))
[:div {:class (str "notification " tclass " has-text-centered")} msg]
)))
;; shortcut
(defn error [msg fail] (notify (str msg " :: " (f/message fail)) "is-danger"))
(defn error
([msg] (notify msg "is-danger"))
([msg fail] (notify (str msg " :: " (f/message fail)) "is-danger")))
(defn render
"render a full webpage using headers, navbar, body and footer"
@ -84,4 +88,30 @@
(resource head)
(conj body (resource footer)))}))
(defn render-template
"render an html from resources using headers, navbar, body and footer"
([body] (render-template nil body))
([account body]
{:headers {"Content-Type"
"text/html; charset=utf-8"}
:body (str "<!DOCTYPE html>\n<html>"
(resource head)
"\n<!-- body -->\n"
(resource body)
"\n<!-- end of body -->\n"
(resource footer))}))
(defn render-html
"render an html from resources using headers, navbar, body and footer"
([body] (render-template nil body))
([account body]
{:headers {"Content-Type"
"text/html; charset=utf-8"}
:body (str "<!DOCTYPE html>\n<html>"
(resource head)
"\n<!-- body -->\n"
body ;; html text string
"\n<!-- end of body -->\n"
(resource footer))}))
(defn render-error [err] (->> "is-danger" (notify err) render))

32
clojure_frontend/src/toaster/views.clj
View File

@ -60,29 +60,31 @@
(web/button "/remove" "\uD83D\uDDD1" (hf/hidden-field "jobid" jobid))]]]
))]]])
(defn dockerfile-upload-post [request config account]
(defn dockerfile-upload [request config account]
(f/attempt-all
[tempfile (s/param request [:file :tempfile])
filename (s/param request [:file :filename])
filesize (s/param request [:file :size])]
(cond
(> filesize 64000)
params (log/spy (:params request))]
(if (> (s/param request [:file :size]) 64000)
;; TODO: put filesize limit in config
(s/notify "File too big in upload (64KB limit)" "is-danger")
:else
(s/error "File too big in upload (64KB limit)")
;; else
(let [file (io/copy tempfile (io/file "/tmp" filename))
path (str "/tmp/" filename)]
(io/delete-file tempfile)
(if (not (.exists (io/file path)))
(s/notify (str "Uploaded file not found: " filename) "is-danger")
(s/error (str "Uploaded file not found: " filename))
;; file is now in 'tmp' var
(f/if-let-ok? [newjob (job/add path config account)]
[:div {:class "container"}
[:h1 {:class "title"} "Job uploaded and added"]
[:p "Log messages:"]
(web/render-yaml newjob)]
;; else when job/add is not-ok
(s/error "Error adding job" newjob)))))
(f/attempt-all
[newjob (job/add path config account)]
[:div {:class "container"}
[:h1 {:class "title"} "Job uploaded and added"]
[:p "Log messages:"]
(web/render-yaml newjob)]
;; else when job/add is not-ok
(f/when-failed [e]
(s/error "Error adding job" e))
))))
(f/when-failed [e]
(s/error "Upload file error" e))))
@ -107,7 +109,7 @@
jobfound (db/query @ring/jobs {:jobid jobid})
r_rmjob (db/delete! @ring/jobs jobid)
r_sync (job/sync_jobs config "-d" jobid)]
(s/notify (str "Job removed: " jobid) "is-primary")
(s/notify (str "Job removed :: " jobid) "is-primary")
(f/when-failed [e]
(s/error "Failure removing job" e))))