diff --git a/clojure_frontend/src/toaster/config.clj b/clojure_frontend/src/toaster/config.clj index 6ac20ef..09061f7 100644 --- a/clojure_frontend/src/toaster/config.clj +++ b/clojure_frontend/src/toaster/config.clj @@ -93,7 +93,7 @@ ;; (s/validate Config conf) ;; (catch Exception ex ;; (f/fail (log/spy :error ["Invalid configuration: " conf ex])))) - (get-in conf path)) + (get-in conf (into [:toaster] path))) (defn load-config [name default] (log/info (str "Loading configuration: " name)) diff --git a/clojure_frontend/src/toaster/handler.clj b/clojure_frontend/src/toaster/handler.clj index 4bdce62..a46d34a 100644 --- a/clojure_frontend/src/toaster/handler.clj +++ b/clojure_frontend/src/toaster/handler.clj @@ -17,153 +17,158 @@ ;; along with this program. If not, see . (ns toaster.handler - (:require - [clojure.string :as str] - [clojure.java.io :as io] - [clojure.data.json :as json] - [compojure.core :refer :all] - [compojure.handler :refer :all] - [compojure.route :as route] - [compojure.response :as response] + (:require + [clojure.string :as str] + [clojure.java.io :as io] + [clojure.data.json :as json] + [compojure.core :refer :all] + [compojure.handler :refer :all] + [compojure.route :as route] + [compojure.response :as response] - [ring.adapter.jetty :refer :all] - [ring.middleware.session :refer :all] - [ring.middleware.accept :refer [wrap-accept]] - [ring.middleware.defaults :refer [wrap-defaults site-defaults]] + [ring.adapter.jetty :refer :all] + [ring.middleware.session :refer :all] + [ring.middleware.accept :refer [wrap-accept]] + [ring.middleware.defaults :refer [wrap-defaults site-defaults]] - [failjure.core :as f] - [taoensso.timbre :as log] - [just-auth.core :as auth] + [failjure.core :as f] + [taoensso.timbre :as log] + [just-auth.core :as auth] - [toaster.session :as s] - [toaster.config :as conf] - [toaster.webpage :as web] - [toaster.ring :as ring] - [toaster.views :as views] - [toaster.jobs :as job]) - (:import java.io.File) - (:gen-class)) + [toaster.session :as s] + [toaster.config :as conf] + [toaster.webpage :as web] + [toaster.ring :as ring] + [toaster.views :as views] + [toaster.jobs :as job]) + (:gen-class)) (defonce config (conf/load-config "toaster" conf/default-settings)) -(defroutes app-routes +(defroutes + app-routes - (GET "/" request (web/render "Hello World!"));; web/readme)) + (GET "/" request (web/render "Hello World!")) ;; web/readme)) ;; NEW ROUTES HERE (GET "/upload" request - (->> (fn [req conf acct] - (web/render acct views/dockerfile-upload-form)) - (s/check request))) + (->> (fn [req conf acct] + (web/render acct views/dockerfile-upload-form)) + (s/check request))) (POST "/dockerfile" request - (->> views/dockerfile-upload-post - (s/check request))) + (->> views/dockerfile-upload-post + (s/check request))) + + (GET "/edit" request + (->> (fn [req conf acct] + (web/render acct views/dockerfile-edit-form)) + (s/check request))) (GET "/list" request - (->> views/list-jobs - (s/check request))) + (->> views/list-jobs + (s/check request))) ;; JUST-AUTH ROUTES (GET "/login" request - (f/attempt-all - [acct (s/check-account request)] - (web/render acct - [:div - [:h1 (str "Already logged in with account: " - (:email acct))] - [:h2 [:a {:href "/logout"} "Logout"]]]) - (f/when-failed [e] - (web/render web/login-form)))) + (f/attempt-all + [acct (s/check-account request)] + (web/render acct + [:div + [:h1 (str "Already logged in with account: " + (:email acct))] + [:h2 [:a {:href "/logout"} "Logout"]]]) + (f/when-failed [e] + (web/render web/login-form)))) (POST "/login" request - (f/attempt-all - [username (s/param request :username) - password (s/param request :password) - logged (auth/sign-in - @ring/auth username password {})] - ;; TODO: pass :ip-address in last argument map - (let [session {:session {:config config - :auth logged}}] - (conj session - (views/list-jobs request config logged))) - ;; (web/render - ;; logged - ;; [:div - ;; [:h1 "Logged in: " username] - ;; views/welcome-menu]))) - (f/when-failed [e] - (web/render-error-page - (str "Login failed: " (f/message e)))))) + (f/attempt-all + [username (s/param request :username) + password (s/param request :password) + logged (auth/sign-in + @ring/auth username password {})] + ;; TODO: pass :ip-address in last argument map + (let [session {:session {:config config + :auth logged}}] + (conj session + (views/list-jobs request config logged))) + ;; (web/render + ;; logged + ;; [:div + ;; [:h1 "Logged in: " username] + ;; views/welcome-menu]))) + (f/when-failed [e] + (web/render-error-page + (str "Login failed: " (f/message e)))))) (GET "/session" request - (-> (:session request) web/render-yaml web/render)) + (-> (:session request) web/render-yaml web/render)) (GET "/logout" request - (conj {:session {:config config}} - (web/render [:h1 "Logged out."]))) + (conj {:session {:config config}} + (web/render [:h1 "Logged out."]))) (GET "/signup" request - (web/render web/signup-form)) + (web/render web/signup-form)) (POST "/signup" request - (f/attempt-all - [name (s/param request :name) - email (s/param request :email) - password (s/param request :password) - repeat-password (s/param request :repeat-password) - activation {:activation-uri - (get-in request [:headers "host"])}] - (web/render - (if (= password repeat-password) - (f/try* - (f/if-let-ok? - [signup (auth/sign-up @ring/auth - name - email - password - activation - [])] - [:div - [:h2 (str "Account created: " - name " <" email ">")] - [:h3 "Account pending activation."]] - (web/render-error + (f/attempt-all + [name (s/param request :name) + email (s/param request :email) + password (s/param request :password) + repeat-password (s/param request :repeat-password) + activation {:activation-uri + (get-in request [:headers "host"])}] + (web/render + (if (= password repeat-password) + (f/try* + (f/if-let-ok? + [signup (auth/sign-up @ring/auth + name + email + password + activation + [])] + [:div + [:h2 (str "Account created: " + name " <" email ">")] + [:h3 "Account pending activation."]] + (web/render-error (str "Failure creating account: " (f/message signup))))) - (web/render-error - "Repeat password didnt match"))) - (f/when-failed [e] - (web/render-error-page - (str "Sign-up failure: " (f/message e)))))) + (web/render-error + "Repeat password didnt match"))) + (f/when-failed [e] + (web/render-error-page + (str "Sign-up failure: " (f/message e)))))) (GET "/activate/:email/:activation-id" [email activation-id :as request] - (let [activation-uri - (str "http://" - (get-in request [:headers "host"]) - "/activate/" email "/" activation-id)] - (web/render - [:div - (f/if-let-failed? - [act (auth/activate-account - @ring/auth email - {:activation-link activation-uri})] - (web/render-error - [:div - [:h1 "Failure activating account"] - [:h2 (f/message act)] - [:p (str "Email: " email " activation-id: " activation-id)]]) - [:h1 (str "Account activated - " email)])]))) + (let [activation-uri + (str "http://" + (get-in request [:headers "host"]) + "/activate/" email "/" activation-id)] + (web/render + [:div + (f/if-let-failed? + [act (auth/activate-account + @ring/auth email + {:activation-link activation-uri})] + (web/render-error + [:div + [:h1 "Failure activating account"] + [:h2 (f/message act)] + [:p (str "Email: " email " activation-id: " activation-id)]]) + [:h1 (str "Account activated - " email)])]))) ;; -- end of JUST-AUTH (POST "/" request - ;; generic endpoint for canceled operations - (web/render (s/check-account request) - [:div {:class (str "alert alert-danger") :role "alert"} - (s/param request :message)])) + ;; generic endpoint for canceled operations + (web/render (s/check-account request) + [:div {:class (str "alert alert-danger") :role "alert"} + (s/param request :message)])) (route/resources "/") (route/not-found (web/render-error-page "Page Not Found")) - ) ;; end of routes + ) ;; end of routes (def app (-> (wrap-defaults app-routes ring/app-defaults) - (wrap-accept {:mime ["text/html"] + (wrap-accept {:mime ["text/html"] ;; preference in language, fallback to english :language ["en" :qs 0.5 "it" :qs 1 @@ -173,7 +178,9 @@ ;; for uberjar (defn -main [] - (println "Starting standalone jetty server on http://localhost:6060") - (run-jetty app {:port 6060 - :host "localhost" - :join? true})) + (println "Starting ring server") + (ring/init ring/app-defaults) + ;(run-jetty app {:port 6060 + ; :host "localhost" + ; :join? true}) + ) diff --git a/clojure_frontend/src/toaster/jobs.clj b/clojure_frontend/src/toaster/jobs.clj index 139a768..e685f19 100644 --- a/clojure_frontend/src/toaster/jobs.clj +++ b/clojure_frontend/src/toaster/jobs.clj @@ -1,40 +1,71 @@ (ns toaster.jobs (:require - [clojure.string :as str] - [clojure.java.io :as io] - [clj-time.core :as time] - [clj-time.coerce :as tc] - [failjure.core :as f] - [taoensso.timbre :as log :refer [debug]] - [me.raynes.conch :as sh :refer [with-programs]] - [toaster.webpage :as web] - [hiccup.form :as hf])) + [clojure.string :as str] + [clojure.java.io :as io] + [clj-time.core :as time] + [clj-time.coerce :as tc] + [clj-storage.db.mongo :refer [create-mongo-store]] + [clj-storage.core :as db] + [failjure.core :as f] + [taoensso.timbre :as log :refer [debug]] + [me.raynes.conch :as sh :refer [with-programs]] + [toaster.webpage :as web] + [toaster.config :refer :all] + [toaster.ring :refer [jobs]] + [hiccup.form :as hf])) + +(defn- ssh-host [config] + (str (q config [:jenkins :user]) "@" (q config [:jenkins :host]))) + + +(defn- sync_jobs [config arg1 arg2] + (with-programs + [ssh] + (try + (-> (ssh "-i" (q config [:jenkins :key]) + (ssh-host config) "sync_jobs.py" + arg1 arg2) + (str/split #"\n")) + (catch Exception e (f/fail (str "ERROR in sync_jobs.py - " (.getMessage e))))))) + +(defn- dockerlint [path] + (with-programs [node] + (try + (node "node_modules/dockerlint/bin/dockerlint.js" path) + (catch Exception e + (f/fail (str "ERROR in dockerlint - " (.getMessage e))))))) + (defn add [path config account] (with-programs [ssh scp node] - (let [tstamp (tc/to-long (time/now)) - jobname (str (:email account) "-vm_amd64-" tstamp) - jobdir (str "/srv/toaster/" jobname)] - (f/attempt-all - [r_lint (node "node_modules/dockerlint/bin/dockerlint.js" path) - r_mkdir (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "mkdir" "-p" jobdir ) - r_scp (scp "-i" "../id_ed25519" path (str "jenkins@sdk.bridge:" jobdir)) - r_job (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-a" jobname)] - {:lint r_lint - :job r_job} - (f/when-failed [e] - (web/render-error-page - (str "Job add failure: " (f/message e)))))))) - + (let [tstamp (tc/to-long (time/now)) + jobname (str (:email account) "-vm_amd64-" tstamp) + jobdir (str "/srv/toaster/" jobname)] + (f/attempt-all + [r_lint (dockerlint path) + r_mkdir (ssh "-i" (q config [:jenkins :key]) + (ssh-host config) "mkdir" "-p" jobdir) + r_scp (scp "-i" (q config [:jenkins :key]) + path (str (ssh-host config) ":" jobdir)) + r_job (log/spy (sync_jobs config "-a" jobname)) + r_store (log/spy (db/store! + @jobs :jobid + (log/spy {:jobid jobname + :email (:email account) + :account (dissoc account :password :activation-link) + :lint (if (.contains r_lint "is OK") true false) + :timestamp tstamp + :type "vm_amd64"})))] + {:lint r_lint + :job r_job} + (f/when-failed [e] + (web/render-error-page + (str "Job add failure: " (f/message e)))))))) + (defn listall [config account] - (with-programs [ssh] - (f/attempt-all - [r_ssh (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-l" (:email account))] - (str/split r_ssh #"\n") - (f/when-failed [e] - (web/render-error-page - (str "Job list failure: " (f/message e))))))) - + (sync_jobs config "-l" (:email account)) + ;;(db/query @jobs {:email (:email account)}) + ) diff --git a/clojure_frontend/src/toaster/ring.clj b/clojure_frontend/src/toaster/ring.clj index 28a0732..14d4fe2 100644 --- a/clojure_frontend/src/toaster/ring.clj +++ b/clojure_frontend/src/toaster/ring.clj @@ -37,39 +37,41 @@ (def db (atom {})) (def accts (atom {})) (def auth (atom {})) +(def jobs (atom {})) (defn init [] - (log/merge-config! {:level :debug - ;; #{:trace :debug :info :warn :error :fatal :report} - - ;; Control log filtering by - ;; namespaces/patterns. Useful for turning off - ;; logging in noisy libraries, etc.: -;; :ns-whitelist ["agiladmin.*" "just-auth.*"] - :ns-blacklist ["org.eclipse.jetty.*" - "org.mongodb.driver.cluster"]}) - + (log/merge-config! + {:level :debug + ;; #{:trace :debug :info :warn :error :fatal :report} + + ;; Control log filtering by + ;; namespaces/patterns. Useful for turning off + ;; logging in noisy libraries, etc.: + ;; i.e: :ns-whitelist ["agiladmin.*" "just-auth.*"] + :ns-blacklist ["org.eclipse.jetty.*" + "org.mongodb.driver.cluster"]}) + ;; ------------------ ;; load configuration (reset! config (conf/load-config (or (System/getenv "toaster_conf") "toaster") conf/default-settings)) - (let [justauth-conf (get-in @config [:toaster :just-auth])] + ;; -------------------------------- + ;; initialize authentication stores + (let [justauth-conf (conf/q @config [:just-auth])] ;; connect database (TODO: take parameters from configuration) (reset! db (get-mongo-db (:mongo-url justauth-conf))) ;; create authentication stores in db (f/attempt-all - [auth-conf (get-in @config [:toaster :just-auth]) - auth-stores (auth-db/create-auth-stores @db)] + [auth-stores (auth-db/create-auth-stores @db)] [(trans/init "lang/auth-en.yml" "lang/english.yaml") (reset! accts auth-stores) - (reset! auth (auth/email-based-authentication + (reset! auth (auth/email-based-authentication auth-stores ;; TODO: replace with email taken from config - (dissoc (:just-auth (:toaster (conf/load-config - "toaster" conf/default-settings))) + (dissoc (conf/q @config [:just-auth]) :mongo-url :mongo-user :mongo-pass) {:criteria #{:email :ip-address} :type :block @@ -80,6 +82,13 @@ (f/when-failed [e] (log/error (str (trans/locale [:init :failure]) " - " (f/message e)))))) + + ;; ---------------------- + ;; initialize jobs stores + (reset! jobs (create-mongo-store @db :job-store)) + + ;; ------------------------------ + ;; log all results worth noticing (log/info (str (trans/locale [:init :success]))) (log/debug @auth)) @@ -87,7 +96,7 @@ (-> site-defaults (assoc-in [:cookies] true) (assoc-in [:security :anti-forgery] - (get-in @config [:webserver :anti-forgery])) + (conf/q @config [:webserver :anti-forgery])) (assoc-in [:security :ssl-redirect] - (get-in @config [:webserver :ssl-redirect])) + (conf/q @config [:webserver :ssl-redirect])) (assoc-in [:security :hsts] true))) diff --git a/clojure_frontend/src/toaster/views.clj b/clojure_frontend/src/toaster/views.clj index 60fec00..f063984 100644 --- a/clojure_frontend/src/toaster/views.clj +++ b/clojure_frontend/src/toaster/views.clj @@ -1,21 +1,22 @@ (ns toaster.views (:require - [clojure.java.io :as io] - [clojure.string :as str] - ;; [clojure.data.json :as json :refer [read-str]] - [toaster.webpage :as web] - [toaster.session :as s] - [toaster.ring :as ring] - [toaster.jobs :as job] - [failjure.core :as f] - [auxiliary.string :refer [strcasecmp]] - [toaster.config :as conf] - [taoensso.timbre :as log :refer [debug]] - [me.raynes.conch :as sh :refer [with-programs]] - [clj-time.core :as time] - [clj-time.coerce :as tc] - [clj-time.local :as tl] - [hiccup.form :as hf])) + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.contrib.humanize :as humanize :refer [datetime]] + ;; [clojure.data.json :as json :refer [read-str]] + [toaster.webpage :as web] + [toaster.session :as s] + [toaster.ring :as ring] + [toaster.jobs :as job] + [failjure.core :as f] + [auxiliary.string :refer [strcasecmp]] + [toaster.config :as conf] + [taoensso.timbre :as log :refer [debug]] + [me.raynes.conch :as sh :refer [with-programs]] + [clj-time.core :as time] + [clj-time.coerce :as tc] + [clj-time.local :as tl] + [hiccup.form :as hf])) (def dockerfile-upload-form [:div {:class "container-fluid"} @@ -23,63 +24,90 @@ [:p " Choose the file in your computer and click 'Submit' to proceed to validation."] [:div {:class "form-group"} - [:form {:action "dockerfile" :method "post" - :class "form-shell" + [:form {:action "dockerfile" :method "post" + :class "form-shell" :enctype "multipart/form-data"} [:fieldset {:class "fieldset btn btn-default btn-file btn-lg"} [:input {:name "file" :type "file"}]] ;; [:fieldset {:class "fieldset-submit"} [:input {:class "btn btn-primary btn-lg" - :id "field-submit" :type "submit" - :name "submit" :value "submit"}]]]]) + :id "field-submit" :type "submit" + :name "submit" :value "submit"}]]]]) + +(def dockerfile-edit-form + [:div {:class "container-fluid"} + [:h1 "Edit your Dockerfile to toast"] + [:div {:class "form-group"} + [:form {:action "dockerfile" :method "post" + :class "form-shell" + :enctype "multipart/form-data"} + [:fieldset {:class "fieldset btn btn-default btn-file btn-lg"} + [:textarea {:name "editor" :id "editor" + :rows 30 :cols 72 } "FROM: dyne/devuan:ascii"] + [:input {:class "btn btn-primary btn-lg" + :id "field-submit" :type "submit" + :name "submit" :value "submit"}]]]] + [:script "var editor = ace.edit(\"editor\"); + editor.setTheme(\"ace/theme/monokai\"); + editor.session.setMode(\"ace/mode/dockerfile\");"]]) + + (def welcome-menu [:div {:class "container-fluid"} [:div {:class "row-fluid"} [:div {:class "row"} [:a {:href "/list"} "List all your toaster jobs"]] [:div {:class "row"} [:a {:href "/upload"} "Upload a new toaster job"]]]]) - + (defn dockerfile-upload-post [request config account] (web/render - account - (let - [tempfile (get-in request [:params :file :tempfile]) - filename (get-in request [:params :file :filename]) - params (:params request)] - (cond - (> (get-in params [:file :size]) 64000) - ;; max upload size in bytes - ;; TODO: put in config - (web/render-error-page params "File too big in upload (64KB limit).") - :else - (let [file (io/copy tempfile (io/file "/tmp" filename)) - path (str "/tmp/" filename)] - (io/delete-file tempfile) - (if (not (.exists (io/file path))) - (web/render-error-page - (log/spy :error - [:h1 (str "Uploaded file not found: " filename)])) - ;; file is now in 'tmp' var - [:div {:class "container-fluid"} - [:h1 "Job uploaded and added"] - [:p "Log messages:"] - (web/render-yaml (job/add path config account))])))))) + account + (let + [tempfile (get-in request [:params :file :tempfile]) + filename (get-in request [:params :file :filename]) + params (:params request)] + (cond + (> (get-in params [:file :size]) 64000) + ;; max upload size in bytes + ;; TODO: put in config + (web/render-error-page params "File too big in upload (64KB limit).") + :else + (let [file (io/copy tempfile (io/file "/tmp" filename)) + path (str "/tmp/" filename)] + (io/delete-file tempfile) + (if (not (.exists (io/file path))) + (web/render-error-page + (log/spy :error + [:h1 (str "Uploaded file not found: " filename)])) + ;; file is now in 'tmp' var + [:div {:class "container-fluid"} + [:h1 "Job uploaded and added"] + [:p "Log messages:"] + (web/render-yaml (job/add path config account))])))))) (defn list-jobs [request config account] - (web/render - account - [:div {:class "container-fluid"} - [:h1 (str "List all toaster jobs for " (:name account))] - [:table {:class "sortable table"} - [:thead nil - [:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]] - [:tbody nil - (for [j (job/listall config account)] - (let [type (-> j (str/split #"-") second) - tstamp (-> j (str/split #"-") last)] - [:tr nil - [:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time)] - [:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/" j)} type]] - [:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j)) - (web/button "/remove" "Remove" (hf/hidden-field "job" j))]] - ))]]])) + (f/attempt-all + [joblist (job/listall config account)] + (web/render + account + [:div {:class "container-fluid"} + [:h1 (str "List all toaster jobs for " (:name account))] + [:table {:class "sortable table"} + [:thead nil + [:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]] + [:tbody nil + (for [j joblist] + (let [type (-> j (str/split #"-") second) + tstamp (-> j (str/split #"-") last)] + [:tr nil + [:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time + humanize/datetime)] + [:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/" + (str/replace j #"@" "AT"))} type]] + [:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j)) + (web/button "/remove" "Remove" (hf/hidden-field "job" j))]] + ))]]]) + (f/when-failed [e] + (web/render-error-page + (str "Job list failure: " (f/message e)))) + )) diff --git a/clojure_frontend/toaster.yaml b/clojure_frontend/toaster.yaml index 064b15d..9de71aa 100644 --- a/clojure_frontend/toaster.yaml +++ b/clojure_frontend/toaster.yaml @@ -2,8 +2,10 @@ webserver: anti-forgery: false ssl-redirect: false -admin: - email: jaromil@dyne.org +jenkins: + host: "bridge.toaster" + user: "jenkins" + key: "../id_ed25519" just-auth: email-server: "mail.dyne.org"