db storage, config fixes and some reformatting

started using intellij cursive for some live debugging, which altered
the indentation of files

clojure_frontend/project.clj

@@ -25,13 +25,17 @@
                  [me.raynes/fs "1.4.6"]
                  [me.raynes/conch "0.8.0"]
                  ;; time from joda-time
-                 [clj-time "0.14.4"]]
+                 [clj-time "0.14.4"]
+                 [clojure-humanize "0.2.2"]]
   :aliases {"test" "midje"}
   :source-paths ["src"]
   :resource-paths ["resources"]
   :plugins [[lein-ring "0.12.4"]]
   :ring    {:init toaster.ring/init
-            :handler toaster.handler/app}
+            :handler toaster.handler/app
+            :nrepl {:start? true
+                    :host "localhost"
+                    :port 40231}}
   :uberwar {:init toaster.ring/init
             :handler toaster.handler/app}
   :mail toaster.handler

clojure_frontend/src/toaster/config.clj

@@ -93,7 +93,7 @@
   ;;   (s/validate Config conf)
   ;;   (catch Exception ex
   ;;     (f/fail (log/spy :error ["Invalid configuration: " conf ex]))))
-  (get-in conf path))
+  (get-in conf (into [:toaster] path)))
 
 (defn load-config [name default]
   (log/info (str "Loading configuration: " name))

clojure_frontend/src/toaster/handler.clj

@@ -41,12 +41,12 @@
     [toaster.ring :as ring]
     [toaster.views :as views]
     [toaster.jobs :as job])
-    (:import java.io.File)
   (:gen-class))
 
 (defonce config (conf/load-config "toaster" conf/default-settings))
 
-(defroutes app-routes
+(defroutes
+  app-routes
 
   (GET "/" request (web/render "Hello World!"))             ;; web/readme))
 
@@ -60,6 +60,11 @@
     (->> views/dockerfile-upload-post
          (s/check request)))
 
+  (GET "/edit" request
+    (->> (fn [req conf acct]
+           (web/render acct views/dockerfile-edit-form))
+         (s/check request)))
+
   (GET "/list" request
     (->> views/list-jobs
          (s/check request)))
@@ -173,7 +178,9 @@
 
 ;; for uberjar
 (defn -main []
-  (println "Starting standalone jetty server on http://localhost:6060")
-  (run-jetty app {:port 6060
-                  :host "localhost"
-                  :join? true}))
+  (println "Starting ring server")
+  (ring/init ring/app-defaults)
+  ;(run-jetty app {:port 6060
+  ;                :host "localhost"
+  ;                :join? true})
+  )

clojure_frontend/src/toaster/jobs.clj

@@ -4,22 +4,58 @@
     [clojure.java.io :as io]
     [clj-time.core :as time]
     [clj-time.coerce :as tc]
+    [clj-storage.db.mongo :refer [create-mongo-store]]
+    [clj-storage.core :as db]
     [failjure.core :as f]
     [taoensso.timbre :as log :refer [debug]]
     [me.raynes.conch :as sh :refer [with-programs]]
     [toaster.webpage :as web]
+    [toaster.config :refer :all]
+    [toaster.ring :refer [jobs]]
     [hiccup.form :as hf]))
 
+(defn- ssh-host [config]
+  (str (q config [:jenkins :user]) "@" (q config [:jenkins :host])))
+
+
+(defn- sync_jobs [config arg1 arg2]
+  (with-programs
+    [ssh]
+    (try
+      (-> (ssh "-i" (q config [:jenkins :key])
+               (ssh-host config) "sync_jobs.py"
+               arg1 arg2)
+          (str/split #"\n"))
+      (catch Exception e (f/fail (str "ERROR in sync_jobs.py - " (.getMessage e)))))))
+
+(defn- dockerlint [path]
+  (with-programs [node]
+                 (try
+                   (node "node_modules/dockerlint/bin/dockerlint.js" path)
+                   (catch Exception e
+                     (f/fail (str "ERROR in dockerlint - " (.getMessage e)))))))
+
+
 (defn add [path config account]
   (with-programs [ssh scp node]
                  (let [tstamp (tc/to-long (time/now))
                        jobname (str (:email account) "-vm_amd64-" tstamp)
                        jobdir (str "/srv/toaster/" jobname)]
                    (f/attempt-all
-       [r_lint  (node "node_modules/dockerlint/bin/dockerlint.js" path)
-        r_mkdir (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "mkdir" "-p" jobdir )
-        r_scp   (scp "-i" "../id_ed25519" path (str "jenkins@sdk.bridge:" jobdir))
-        r_job   (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-a" jobname)]
+                     [r_lint (dockerlint path)
+                      r_mkdir (ssh "-i" (q config [:jenkins :key])
+                                   (ssh-host config) "mkdir" "-p" jobdir)
+                      r_scp (scp "-i" (q config [:jenkins :key])
+                                 path (str (ssh-host config) ":" jobdir))
+                      r_job (log/spy (sync_jobs config "-a" jobname))
+                      r_store (log/spy (db/store!
+                                         @jobs :jobid
+                                         (log/spy {:jobid     jobname
+                                                   :email     (:email account)
+                                                   :account   (dissoc account :password :activation-link)
+                                                   :lint      (if (.contains r_lint "is OK") true false)
+                                                   :timestamp tstamp
+                                                   :type      "vm_amd64"})))]
                      {:lint r_lint
                       :job  r_job}
                      (f/when-failed [e]
@@ -28,13 +64,8 @@
 
 
 (defn listall [config account]
-  (with-programs [ssh]
-    (f/attempt-all
-     [r_ssh   (ssh "-i" "../id_ed25519" "jenkins@sdk.bridge" "sync_jobs.py" "-l" (:email account))]
-     (str/split r_ssh #"\n")
-     (f/when-failed [e]
-       (web/render-error-page
-        (str "Job list failure: " (f/message e)))))))
-
+  (sync_jobs config "-l" (:email account))
+  ;;(db/query @jobs {:email (:email account)})
+  )
 
 

clojure_frontend/src/toaster/ring.clj

@@ -37,39 +37,41 @@
 (def db     (atom {}))
 (def accts  (atom {}))
 (def auth   (atom {}))
+(def jobs   (atom {}))
 
 (defn init []
-  (log/merge-config! {:level :debug
+  (log/merge-config!
+   {:level :debug
     ;; #{:trace :debug :info :warn :error :fatal :report}
     
     ;; Control log filtering by
     ;; namespaces/patterns. Useful for turning off
     ;; logging in noisy libraries, etc.:
-;;                      :ns-whitelist  ["agiladmin.*" "just-auth.*"]
+    ;; i.e: :ns-whitelist  ["agiladmin.*" "just-auth.*"]
     :ns-blacklist  ["org.eclipse.jetty.*"
                     "org.mongodb.driver.cluster"]})
-
+  ;; ------------------
   ;; load configuration
   (reset! config (conf/load-config
                   (or (System/getenv "toaster_conf") "toaster")
                   conf/default-settings))
 
-  (let [justauth-conf (get-in @config [:toaster :just-auth])]
+  ;; --------------------------------
+  ;; initialize authentication stores
+  (let [justauth-conf (conf/q @config [:just-auth])]
     ;; connect database (TODO: take parameters from configuration)
     (reset! db (get-mongo-db (:mongo-url justauth-conf)))
 
     ;; create authentication stores in db
     (f/attempt-all
-     [auth-conf   (get-in @config [:toaster :just-auth])
-      auth-stores (auth-db/create-auth-stores @db)]
+     [auth-stores (auth-db/create-auth-stores @db)]
 
      [(trans/init "lang/auth-en.yml" "lang/english.yaml")
       (reset! accts auth-stores)
       (reset! auth (auth/email-based-authentication
                     auth-stores
                     ;; TODO: replace with email taken from config
-                    (dissoc (:just-auth (:toaster (conf/load-config
-                                                    "toaster" conf/default-settings)))
+                    (dissoc (conf/q @config [:just-auth])
                             :mongo-url :mongo-user :mongo-pass)
                     {:criteria #{:email :ip-address} 
                      :type :block
@@ -80,6 +82,13 @@
      (f/when-failed [e]
        (log/error (str (trans/locale [:init :failure])
                        " - " (f/message e))))))
+
+  ;; ----------------------
+  ;; initialize jobs stores
+  (reset! jobs (create-mongo-store @db :job-store))
+
+  ;; ------------------------------
+  ;; log all results worth noticing
   (log/info (str (trans/locale [:init :success])))
   (log/debug @auth))
 
@@ -87,7 +96,7 @@
     (-> site-defaults
         (assoc-in [:cookies] true)
         (assoc-in [:security :anti-forgery]
-                  (get-in @config [:webserver :anti-forgery]))
+                  (conf/q @config [:webserver :anti-forgery]))
         (assoc-in [:security :ssl-redirect]
-                  (get-in @config [:webserver :ssl-redirect]))
+                  (conf/q @config [:webserver :ssl-redirect]))
         (assoc-in [:security :hsts] true)))

clojure_frontend/src/toaster/views.clj

@@ -2,6 +2,7 @@
   (:require
     [clojure.java.io :as io]
     [clojure.string :as str]
+    [clojure.contrib.humanize :as humanize :refer [datetime]]
     ;;   [clojure.data.json :as json :refer [read-str]]
     [toaster.webpage :as web]
     [toaster.session :as s]
@@ -33,6 +34,25 @@ proceed to validation."]
               :id    "field-submit" :type "submit"
               :name  "submit" :value "submit"}]]]])
 
+(def dockerfile-edit-form
+  [:div {:class "container-fluid"}
+   [:h1 "Edit your Dockerfile to toast"]
+   [:div {:class "form-group"}
+    [:form {:action  "dockerfile" :method "post"
+            :class   "form-shell"
+            :enctype "multipart/form-data"}
+     [:fieldset {:class "fieldset btn btn-default btn-file btn-lg"}
+      [:textarea {:name "editor" :id "editor"
+                  :rows 30 :cols 72 } "FROM: dyne/devuan:ascii"]
+      [:input {:class "btn btn-primary btn-lg"
+               :id    "field-submit" :type "submit"
+               :name  "submit" :value "submit"}]]]]
+   [:script "var editor = ace.edit(\"editor\");
+      editor.setTheme(\"ace/theme/monokai\");
+      editor.session.setMode(\"ace/mode/dockerfile\");"]])
+
+
+
 (def welcome-menu
   [:div {:class "container-fluid"}
    [:div {:class "row-fluid"}
@@ -66,6 +86,8 @@ proceed to validation."]
              (web/render-yaml (job/add path config account))]))))))
 
 (defn list-jobs [request config account]
+  (f/attempt-all
+    [joblist (job/listall config account)]
     (web/render
       account
       [:div {:class "container-fluid"}
@@ -74,12 +96,18 @@ proceed to validation."]
         [:thead nil
          [:tr nil [:th nil "Date"] [:th nil "Type"] [:th nil "Actions"]]]
         [:tbody nil
-      (for [j (job/listall config account)]
+         (for [j joblist]
            (let [type (-> j (str/split #"-") second)
                  tstamp (-> j (str/split #"-") last)]
              [:tr nil
-           [:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time)]
-           [:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/" j)} type]]
+              [:td {:class "date"} (-> tstamp Long/valueOf tc/from-long tl/to-local-date-time
+                                       humanize/datetime)]
+              [:td {:class "job"} [:a {:href (str "https://sdk.dyne.org:4443/view/web-sdk-builds/job/"
+                                                  (str/replace j #"@" "AT"))} type]]
               [:td {:class "start-job"} (web/button "/start" "Start" (hf/hidden-field "job" j))
                (web/button "/remove" "Remove" (hf/hidden-field "job" j))]]
-          ))]]]))
+             ))]]])
+    (f/when-failed [e]
+                   (web/render-error-page
+                     (str "Job list failure: " (f/message e))))
+    ))

clojure_frontend/toaster.yaml

@@ -2,8 +2,10 @@ webserver:
   anti-forgery: false
   ssl-redirect: false
 
-admin:
-  email: jaromil@dyne.org
+jenkins:
+  host: "bridge.toaster"
+  user: "jenkins"
+  key: "../id_ed25519"
 
 just-auth:
   email-server: "mail.dyne.org"