db storage, config fixes and some reformatting

started using intellij cursive for some live debugging, which altered
the indentation of files
This commit is contained in:
Jaromil 2018-10-08 12:49:19 +02:00
parent a7deb67864
commit c277e7a6a8
6 changed files with 306 additions and 229 deletions

View File

@ -93,7 +93,7 @@
;; (s/validate Config conf)
;; (catch Exception ex
;; (f/fail (log/spy :error ["Invalid configuration: " conf ex]))))
(get-in conf path))
(get-in conf (into [:toaster] path)))
(defn load-config [name default]
(log/info (str "Loading configuration: " name))

View File

@ -17,153 +17,158 @@
;; along with this program. If not, see <http://www.gnu.org/licenses/>.
(ns toaster.handler
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clojure.data.json :as json]
[compojure.core :refer :all]
[compojure.handler :refer :all]
[compojure.route :as route]
[compojure.response :as response]
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clojure.data.json :as json]
[compojure.core :refer :all]
[compojure.handler :refer :all]
[compojure.route :as route]
[compojure.response :as response]
[ring.adapter.jetty :refer :all]
[ring.middleware.session :refer :all]
[ring.middleware.accept :refer [wrap-accept]]
[ring.middleware.defaults :refer [wrap-defaults site-defaults]]
[ring.adapter.jetty :refer :all]
[ring.middleware.session :refer :all]
[ring.middleware.accept :refer [wrap-accept]]
[ring.middleware.defaults :refer [wrap-defaults site-defaults]]
[failjure.core :as f]
[taoensso.timbre :as log]
[just-auth.core :as auth]
[failjure.core :as f]
[taoensso.timbre :as log]
[just-auth.core :as auth]
[toaster.session :as s]
[toaster.config :as conf]
[toaster.webpage :as web]
[toaster.ring :as ring]
[toaster.views :as views]
[toaster.jobs :as job])
(:import java.io.File)
(:gen-class))
[toaster.session :as s]
[toaster.config :as conf]
[toaster.webpage :as web]
[toaster.ring :as ring]
[toaster.views :as views]
[toaster.jobs :as job])
(:gen-class))
(defonce config (conf/load-config "toaster" conf/default-settings))
(defroutes app-routes
(defroutes
app-routes
(GET "/" request (web/render "Hello World!"));; web/readme))
(GET "/" request (web/render "Hello World!")) ;; web/readme))
;; NEW ROUTES HERE
(GET "/upload" request
(->> (fn [req conf acct]
(web/render acct views/dockerfile-upload-form))
(s/check request)))
(->> (fn [req conf acct]
(web/render acct views/dockerfile-upload-form))
(s/check request)))
(POST "/dockerfile" request
(->> views/dockerfile-upload-post
(s/check request)))
(->> views/dockerfile-upload-post
(s/check request)))
(GET "/edit" request
(->> (fn [req conf acct]
(web/render acct views/dockerfile-edit-form))
(s/check request)))
(GET "/list" request
(->> views/list-jobs
(s/check request)))
(->> views/list-jobs
(s/check request)))
;; JUST-AUTH ROUTES
(GET "/login" request
(f/attempt-all
[acct (s/check-account request)]
(web/render acct
[:div
[:h1 (str "Already logged in with account: "
(:email acct))]
[:h2 [:a {:href "/logout"} "Logout"]]])
(f/when-failed [e]
(web/render web/login-form))))
(f/attempt-all
[acct (s/check-account request)]
(web/render acct
[:div
[:h1 (str "Already logged in with account: "
(:email acct))]
[:h2 [:a {:href "/logout"} "Logout"]]])
(f/when-failed [e]
(web/render web/login-form))))
(POST "/login" request
(f/attempt-all
[username (s/param request :username)
password (s/param request :password)
logged (auth/sign-in
@ring/auth username password {})]
;; TODO: pass :ip-address in last argument map
(let [session {:session {:config config
:auth logged}}]
(conj session
(views/list-jobs request config logged)))
;; (web/render
;; logged
;; [:div
;; [:h1 "Logged in: " username]
;; views/welcome-menu])))
(f/when-failed [e]
(web/render-error-page
(str "Login failed: " (f/message e))))))
(f/attempt-all
[username (s/param request :username)
password (s/param request :password)
logged (auth/sign-in
@ring/auth username password {})]
;; TODO: pass :ip-address in last argument map
(let [session {:session {:config config
:auth logged}}]
(conj session
(views/list-jobs request config logged)))
;; (web/render
;; logged
;; [:div
;; [:h1 "Logged in: " username]
;; views/welcome-menu])))
(f/when-failed [e]
(web/render-error-page
(str "Login failed: " (f/message e))))))
(GET "/session" request
(-> (:session request) web/render-yaml web/render))
(-> (:session request) web/render-yaml web/render))
(GET "/logout" request
(conj {:session {:config config}}
(web/render [:h1 "Logged out."])))
(conj {:session {:config config}}
(web/render [:h1 "Logged out."])))
(GET "/signup" request
(web/render web/signup-form))
(web/render web/signup-form))
(POST "/signup" request
(f/attempt-all
[name (s/param request :name)
email (s/param request :email)
password (s/param request :password)
repeat-password (s/param request :repeat-password)
activation {:activation-uri
(get-in request [:headers "host"])}]
(web/render
(if (= password repeat-password)
(f/try*
(f/if-let-ok?
[signup (auth/sign-up @ring/auth
name
email
password
activation
[])]
[:div
[:h2 (str "Account created: "
name " &lt;" email "&gt;")]
[:h3 "Account pending activation."]]
(web/render-error
(f/attempt-all
[name (s/param request :name)
email (s/param request :email)
password (s/param request :password)
repeat-password (s/param request :repeat-password)
activation {:activation-uri
(get-in request [:headers "host"])}]
(web/render
(if (= password repeat-password)
(f/try*
(f/if-let-ok?
[signup (auth/sign-up @ring/auth
name
email
password
activation
[])]
[:div
[:h2 (str "Account created: "
name " &lt;" email "&gt;")]
[:h3 "Account pending activation."]]
(web/render-error
(str "Failure creating account: "
(f/message signup)))))
(web/render-error
"Repeat password didnt match")))
(f/when-failed [e]
(web/render-error-page
(str "Sign-up failure: " (f/message e))))))
(web/render-error
"Repeat password didnt match")))
(f/when-failed [e]
(web/render-error-page
(str "Sign-up failure: " (f/message e))))))
(GET "/activate/:email/:activation-id"
[email activation-id :as request]
(let [activation-uri
(str "http://"
(get-in request [:headers "host"])
"/activate/" email "/" activation-id)]
(web/render
[:div
(f/if-let-failed?
[act (auth/activate-account
@ring/auth email
{:activation-link activation-uri})]
(web/render-error
[:div
[:h1 "Failure activating account"]
[:h2 (f/message act)]
[:p (str "Email: " email " activation-id: " activation-id)]])
[:h1 (str "Account activated - " email)])])))
(let [activation-uri
(str "http://"
(get-in request [:headers "host"])
"/activate/" email "/" activation-id)]
(web/render
[:div
(f/if-let-failed?
[act (auth/activate-account
@ring/auth email
{:activation-link activation-uri})]
(web/render-error
[:div
[:h1 "Failure activating account"]
[:h2 (f/message act)]
[:p (str "Email: " email " activation-id: " activation-id)]])
[:h1 (str "Account activated - " email)])])))
;; -- end of JUST-AUTH
(POST "/" request
;; generic endpoint for canceled operations
(web/render (s/check-account request)
[:div {:class (str "alert alert-danger") :role "alert"}
(s/param request :message)]))
;; generic endpoint for canceled operations
(web/render (s/check-account request)
[:div {:class (str "alert alert-danger") :role "alert"}
(s/param request :message)]))
(route/resources "/")
(route/not-found (web/render-error-page "Page Not Found"))
) ;; end of routes
) ;; end of routes
(def app
(-> (wrap-defaults app-routes ring/app-defaults)
(wrap-accept {:mime ["text/html"]
(wrap-accept {:mime ["text/html"]
;; preference in language, fallback to english
:language ["en" :qs 0.5
"it" :qs 1
@ -173,7 +178,9 @@
;; for uberjar
(defn -main []
(println "Starting standalone jetty server on http://localhost:6060")
(run-jetty app {:port 6060
:host "localhost"
:join? true}))
(println "Starting ring server")
(ring/init ring/app-defaults)
;(run-jetty app {:port 6060
; :host "localhost"
; :join? true})
)

View File

@ -1,40 +1,71 @@
(ns toaster.jobs
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.webpage :as web]
[hiccup.form :as hf]))
[clojure.string :as str]
[clojure.java.io :as io]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-storage.db.mongo :refer [create-mongo-store]]
[clj-storage.core :as db]
[failjure.core :as f]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[toaster.webpage :as web]
[toaster.config :refer :all]
[toaster.ring :refer [jobs]]
[hiccup.form :as hf]))
(defn- ssh-host [config]
(str (q config [:jenkins :user]) "@" (q config [:jenkins :host])))
(defn- sync_jobs [config arg1 arg2]
(with-programs
[ssh]
(try
(-> (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "sync_jobs.py"
arg1 arg2)
(str/split #"\n"))
(catch Exception e (f/fail (str "ERROR in sync_jobs.py - " (.getMessage e)))))))
(defn- dockerlint [path]
(with-programs [node]
(try
(node "node_modules/dockerlint/bin/dockerlint.js" path)
(catch Exception e
(f/fail (str "ERROR in dockerlint - " (.getMessage e)))))))
(defn add [path config account]
(with-programs [ssh scp node]
(let [tstamp (tc/to-long (time/now))
jobname (str (:email account) "-vm_amd64-" tstamp)
jobdir (str "/srv/toaster/" jobname)]
(f/attempt-all
[r_lint (node "node_modules/dockerlint/bin/dockerlint.js" path)
r_mkdir (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "mkdir" "-p" jobdir )
r_scp (scp "-i" "../id_ed25519" path (str "jenkins@sdk.bridge:" jobdir))
r_job (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-a" jobname)]
{:lint r_lint
:job r_job}
(f/when-failed [e]
(web/render-error-page
(str "Job add failure: " (f/message e))))))))
(let [tstamp (tc/to-long (time/now))
jobname (str (:email account) "-vm_amd64-" tstamp)
jobdir (str "/srv/toaster/" jobname)]
(f/attempt-all
[r_lint (dockerlint path)
r_mkdir (ssh "-i" (q config [:jenkins :key])
(ssh-host config) "mkdir" "-p" jobdir)
r_scp (scp "-i" (q config [:jenkins :key])
path (str (ssh-host config) ":" jobdir))
r_job (log/spy (sync_jobs config "-a" jobname))
r_store (log/spy (db/store!
@jobs :jobid
(log/spy {:jobid jobname
:email (:email account)
:account (dissoc account :password :activation-link)
:lint (if (.contains r_lint "is OK") true false)
:timestamp tstamp
:type "vm_amd64"})))]
{:lint r_lint
:job r_job}
(f/when-failed [e]
(web/render-error-page
(str "Job add failure: " (f/message e))))))))
(defn listall [config account]
(with-programs [ssh]
(f/attempt-all
[r_ssh (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-l" (:email account))]
(str/split r_ssh #"\n")
(f/when-failed [e]
(web/render-error-page
(str "Job list failure: " (f/message e)))))))
(sync_jobs config "-l" (:email account))
;;(db/query @jobs {:email (:email account)})
)

View File

@ -37,39 +37,41 @@
(def db (atom {}))
(def accts (atom {}))
(def auth (atom {}))
(def jobs (atom {}))
(defn init []
(log/merge-config! {:level :debug
;; #{:trace :debug :info :warn :error :fatal :report}
;; Control log filtering by
;; namespaces/patterns. Useful for turning off
;; logging in noisy libraries, etc.:
;; :ns-whitelist ["agiladmin.*" "just-auth.*"]
:ns-blacklist ["org.eclipse.jetty.*"
"org.mongodb.driver.cluster"]})
(log/merge-config!
{:level :debug
;; #{:trace :debug :info :warn :error :fatal :report}
;; Control log filtering by
;; namespaces/patterns. Useful for turning off
;; logging in noisy libraries, etc.:
;; i.e: :ns-whitelist ["agiladmin.*" "just-auth.*"]
:ns-blacklist ["org.eclipse.jetty.*"
"org.mongodb.driver.cluster"]})
;; ------------------
;; load configuration
(reset! config (conf/load-config
(or (System/getenv "toaster_conf") "toaster")
conf/default-settings))
(let [justauth-conf (get-in @config [:toaster :just-auth])]
;; --------------------------------
;; initialize authentication stores
(let [justauth-conf (conf/q @config [:just-auth])]
;; connect database (TODO: take parameters from configuration)
(reset! db (get-mongo-db (:mongo-url justauth-conf)))
;; create authentication stores in db
(f/attempt-all
[auth-conf (get-in @config [:toaster :just-auth])
auth-stores (auth-db/create-auth-stores @db)]
[auth-stores (auth-db/create-auth-stores @db)]
[(trans/init "lang/auth-en.yml" "lang/english.yaml")
(reset! accts auth-stores)
(reset! auth (auth/email-based-authentication
(reset! auth (auth/email-based-authentication
auth-stores
;; TODO: replace with email taken from config
(dissoc (:just-auth (:toaster (conf/load-config
"toaster" conf/default-settings)))
(dissoc (conf/q @config [:just-auth])
:mongo-url :mongo-user :mongo-pass)
{:criteria #{:email :ip-address}
:type :block
@ -80,6 +82,13 @@
(f/when-failed [e]
(log/error (str (trans/locale [:init :failure])
" - " (f/message e))))))
;; ----------------------
;; initialize jobs stores
(reset! jobs (create-mongo-store @db :job-store))
;; ------------------------------
;; log all results worth noticing
(log/info (str (trans/locale [:init :success])))
(log/debug @auth))
@ -87,7 +96,7 @@
(-> site-defaults
(assoc-in [:cookies] true)
(assoc-in [:security :anti-forgery]
(get-in @config [:webserver :anti-forgery]))
(conf/q @config [:webserver :anti-forgery]))
(assoc-in [:security :ssl-redirect]
(get-in @config [:webserver :ssl-redirect]))
(conf/q @config [:webserver :ssl-redirect]))
(assoc-in [:security :hsts] true)))

View File

@ -1,21 +1,22 @@
(ns toaster.views
(:require
[clojure.java.io :as io]
[clojure.string :as str]
;; [clojure.data.json :as json :refer [read-str]]
[toaster.webpage :as web]
[toaster.session :as s]
[toaster.ring :as ring]
[toaster.jobs :as job]
[failjure.core :as f]
[auxiliary.string :refer [strcasecmp]]
[toaster.config :as conf]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-time.local :as tl]
[hiccup.form :as hf]))
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.contrib.humanize :as humanize :refer [datetime]]
;; [clojure.data.json :as json :refer [read-str]]
[toaster.webpage :as web]
[toaster.session :as s]
[toaster.ring :as ring]
[toaster.jobs :as job]
[failjure.core :as f]
[auxiliary.string :refer [strcasecmp]]
[toaster.config :as conf]
[taoensso.timbre :as log :refer [debug]]
[me.raynes.conch :as sh :refer [with-programs]]
[clj-time.core :as time]
[clj-time.coerce :as tc]
[clj-time.local :as tl]
[hiccup.form :as hf]))
(def dockerfile-upload-form
[:div {:class "container-fluid"}
@ -23,63 +24,90 @@
[:p " Choose the file in your computer and click 'Submit' to
proceed to validation."]
[:div {:class "form-group"}
[:form {:action "dockerfile" :method "post"
:class "form-shell"
[:form {:action "dockerfile" :method "post"
:class "form-shell"
:enctype "multipart/form-data"}
[:fieldset {:class "fieldset btn btn-default btn-file btn-lg"}
[:input {:name "file" :type "file"}]]
;; [:fieldset {:class "fieldset-submit"}
[:input {:class "btn btn-primary btn-lg"
:id "field-submit" :type "submit"
:name "submit" :value "submit"}]]]])
:id "field-submit" :type "submit"
:name "submit" :value "submit"}]]]])
(def dockerfile-edit-form
[:div {:class "container-fluid"}
[:h1 "Edit your Dockerfile to toast"]
[:div {:class "form-group"}
[:form {:action "dockerfile" :method "post"
:class "form-shell"
:enctype "multipart/form-data"}
[:fieldset {:class "fieldset btn btn-default btn-file btn-lg"}
[:textarea {:name "editor" :id "editor"
:rows 30 :cols 72 } "FROM: dyne/devuan:ascii"]
[:input {:class "btn btn-primary btn-lg"
:id "field-submit" :type "submit"
:name "submit" :value "submit"}]]]]
[:script "var editor = ace.edit(\"editor\");
editor.setTheme(\"ace/theme/monokai\");
editor.session.setMode(\"ace/mode/dockerfile\");"]])
(def welcome-menu
[:div {:class "container-fluid"}
[:div {:class "row-fluid"}
[:div {:class "row"} [:a {:href "/list"} "List all your toaster jobs"]]
[:div {:class "row"} [:a {:href "/upload"} "Upload a new toaster job"]]]])
(defn dockerfile-upload-post [request config account]
(web/render
account
(let
[tempfile (get-in request [:params :file :tempfile])
filename (get-in request [:params :file :filename])
params (:params request)]
(cond
(> (get-in params [:file :size]) 64000)
;; max upload size in bytes
;; TODO: put in config
(web/render-error-page params "File too big in upload (64KB limit).")
:else
(let [file (io/copy tempfile (io/file "/tmp" filename))
path (str "/tmp/" filename)]
(io/delete-file tempfile)
(if (not (.exists (io/file path)))
(web/render-error-page
(log/spy :error
[:h1 (str "Uploaded file not found: " filename)]))
;; file is now in 'tmp' var
[:div {:class "container-fluid"}
[:h1 "Job uploaded and added"]
[:p "Log messages:"]
(web/render-yaml (job/add path config account))]))))))
account
(let
[tempfile (get-in request [:params :file :tempfile])
filename (get-in request [:params :file :filename])
params (:params request)]
(cond
(> (get-in params [:file :size]) 64000)
;; max upload size in bytes
;; TODO: put in config
(web/render-error-page params "File too big in upload (64KB limit).")
:else
(let [file (io/copy tempfile (io/file "/tmp" filename))
path (str "/tmp/" filename)]
(io/delete-file tempfile)
(if (not (.exists (io/file path)))
(web/render-error-page
(log/spy :error
[:h1 (str "Uploaded file not found: " filename)]))
;; file is now in 'tmp' var
[:div {:class "container-fluid"}
[:h1 "Job uploaded and added"]
[:p "Log messages:"]
(web/render-yaml (job/add path config account))]))))))
(defn list-jobs [request config account]
(web/render
account
[:div {:class "container-fluid"}
[:h1 (str "List all toaster jobs for " (:name account))]
[:table {:class "sortable table"}
[:thead nil
[:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]]
[:tbody nil
(for [j (job/listall config account)]
(let [type (-> j (str/split #"-") second)
tstamp (-> j (str/split #"-") last)]
[:tr nil
[:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time)]
[:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/" j)} type]]
[:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j))
(web/button "/remove" "Remove" (hf/hidden-field "job" j))]]
))]]]))
(f/attempt-all
[joblist (job/listall config account)]
(web/render
account
[:div {:class "container-fluid"}
[:h1 (str "List all toaster jobs for " (:name account))]
[:table {:class "sortable table"}
[:thead nil
[:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]]
[:tbody nil
(for [j joblist]
(let [type (-> j (str/split #"-") second)
tstamp (-> j (str/split #"-") last)]
[:tr nil
[:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time
humanize/datetime)]
[:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/"
(str/replace j #"@" "AT"))} type]]
[:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j))
(web/button "/remove" "Remove" (hf/hidden-field "job" j))]]
))]]])
(f/when-failed [e]
(web/render-error-page
(str "Job list failure: " (f/message e))))
))

View File

@ -2,8 +2,10 @@ webserver:
anti-forgery: false
ssl-redirect: false
admin:
email: jaromil@dyne.org
jenkins:
host: "bridge.toaster"
user: "jenkins"
key: "../id_ed25519"
just-auth:
email-server: "mail.dyne.org"