From 01be81472678bf17971ea1e73a8b83a1f1cba29d Mon Sep 17 00:00:00 2001 From: Tyler Date: Fri, 29 May 2020 17:50:20 -0400 Subject: [PATCH] Feat/rss (#14) * Feat: add clean anchor and internal link handler. (#13) Add: empty footnote ref/def css classes. Add: add sluggified id's to headlines. Fix: add code child to get-headline-helper. * Clean: project.clj and cli defaults. * Clean: cljfmt-fix. --- clojure/project.clj | 5 ++- clojure/reflection.json | 6 +++ .../resources/firn/_firn_starter/config.edn | 9 ++++- clojure/src/firn/build.clj | 40 +++++++++++-------- clojure/src/firn/config.clj | 28 ++++++------- clojure/src/firn/core.clj | 3 +- clojure/src/firn/dirwatch.clj | 22 +++++----- clojure/src/firn/file.clj | 40 ++++++++++++++++--- clojure/src/firn/layout.clj | 3 +- clojure/src/firn/markup.clj | 15 ++++--- clojure/src/firn/org.clj | 6 +-- clojure/src/firn/server.clj | 8 ++-- clojure/src/firn/util.clj | 33 ++++++++++----- clojure/test/firn/demo_org/file-footnotes.org | 4 +- clojure/test/firn/layout_test.clj | 1 - clojure/test/firn/markup_test.clj | 1 - clojure/test/firn/org_test.clj | 1 - clojure/test/firn/util_test.clj | 7 +++- 18 files changed, 145 insertions(+), 87 deletions(-) diff --git a/clojure/project.clj b/clojure/project.clj index b54ec106..07a51920 100644 --- a/clojure/project.clj +++ b/clojure/project.clj @@ -1,10 +1,11 @@ (defproject firn "0.1.0-SNAPSHOT" - :description "FIXME: write description" - :url "http://example.com/FIXME" + :description "Static Site Generator for Org Mode" + :url "https://github.com/theiceshelf/firn" :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0" :url "https://www.eclipse.org/legal/epl-2.0/"} :dependencies [[borkdude/sci "0.0.13-alpha.19"] [cheshire "5.10.0"] + [clj-rss "0.2.5"] [hiccup "1.0.5"] [http-kit "2.3.0"] ;; [juxt/dirwatch "0.2.5"] ;; vendored diff --git a/clojure/reflection.json b/clojure/reflection.json index 4b81cb1a..7e8a8896 100644 --- a/clojure/reflection.json +++ b/clojure/reflection.json @@ -1,5 +1,11 @@ [ { "name" : "java.nio.file.StandardWatchEventKinds$StdWatchEventKind[]" + }, + { + "name": "com.sun.xml.internal.stream.XMLOutputFactoryImpl", + "allPublicConstructors": true, + "allPublicFields": true, + "allPublicMethods": true } ] diff --git a/clojure/resources/firn/_firn_starter/config.edn b/clojure/resources/firn/_firn_starter/config.edn index f392a9e5..183fe1c6 100644 --- a/clojure/resources/firn/_firn_starter/config.edn +++ b/clojure/resources/firn/_firn_starter/config.edn @@ -1,2 +1,7 @@ -{:dir-data "data" ; org attachments/files to get copied into _site. - :ignored-dirs ["priv"]} ; Directories to ignore org files in. +{ + :dir-data "data" ; org attachments/files to get copied into _site. + :ignored-dirs ["priv"] ; Directories to ignore org files in. + :site-url "" ; Root level url + :site-title "" ; Used for RSS. + :site-desc "" ; Used for RSS. + :enable-rss? true} ; defaults to true; creating feed.xml in your _site. diff --git a/clojure/src/firn/build.clj b/clojure/src/firn/build.clj index 7da1cc3a..341271fc 100644 --- a/clojure/src/firn/build.clj +++ b/clojure/src/firn/build.clj @@ -10,10 +10,10 @@ (set! *warn-on-reflection* true) (def default-files - ["layouts/default.clj" - "partials/head.clj" - "config.edn" - "static/css/main.css"]) + ["layouts/default.clj" + "partials/head.clj" + "config.edn" + "static/css/main.css"]) (defn new-site "Creates the folders needed for a new site in your wiki directory. @@ -31,14 +31,16 @@ (io/make-parents (:out-name f)) (spit (:out-name f) (:contents f))))))) - - (defn setup "Creates folders for output, slurps in layouts and partials. NOTE: should slurp/mkdir/copy-dir be wrapped in try-catches? if-err handling?" - [{:keys [dir-site dir-files dir-site-data - dir-data dir-site-static dir-static] :as config}] + [{:keys [dir-site + dir-files + dir-site-data + dir-data + dir-site-static + dir-static] :as config}] (when-not (fs/exists? (config :dir-firn)) (new-site config)) (fs/mkdir dir-site) ;; make _site @@ -49,10 +51,11 @@ (fs/delete-dir dir-site-static) (fs/copy-dir dir-static dir-site-static) - (assoc config - :org-files (u/find-files-by-ext dir-files "org") - :layouts (file/read-clj :layouts config) - :partials (file/read-clj :partials config))) + (let [org-files (u/find-files-by-ext dir-files "org") + layouts (file/read-clj :layouts config) + partials (file/read-clj :partials config)] + + (assoc config :org-files org-files :layouts layouts :partials partials))) (defn write-files "Takes a config, of which we can presume has :processed-files. @@ -62,12 +65,15 @@ (let [out-file-name (str (config :dir-site) (f :path-web) ".html")] (when-not (file/is-private? config f) (io/make-parents out-file-name) - (spit out-file-name (f :as-html)))))) + (spit out-file-name (f :as-html))))) + config) (defn all-files "Processes all files in the org-directory" [{:keys [dir]}] - (let [config (setup (config/prepare dir))] - (->> config - file/process-all - write-files))) + (let [config (setup (config/prepare dir)) + {:keys [enable-rss?]} config] + (cond->> config + true file/process-all + enable-rss? file/write-rss-file! + true write-files))) diff --git a/clojure/src/firn/config.clj b/clojure/src/firn/config.clj index da589693..25983160 100644 --- a/clojure/src/firn/config.clj +++ b/clojure/src/firn/config.clj @@ -5,21 +5,21 @@ [sci.core :as sci])) (def starting-config - {:dir-data "data" ; org attachments/files to get copied into _site. - :dir-files nil ; where org content lives. - :dir-layouts "" ; where layouts are stored. - :dir-partials "" ; where partials are stored. - :dir-site "" ; the root dir of the compiled firn site. - :dirname-files nil ; the name of directory where firn is run. - :ignored-dirs ["priv"] ; Directories to ignore org files in. - :layouts {} ; layouts loaded into memory - :partials {} ; partials loaded into memory + {:dir-data "data" ; org attachments/files to get copied into _site. + :dir-files nil ; where org content lives. + :dir-layouts "" ; where layouts are stored. + :dir-partials "" ; where partials are stored. + :dir-site "" ; the root dir of the compiled firn site. + :dirname-files nil ; the name of directory where firn is run. + :site-url "" ; Root level url + :site-title "" ; Used for RSS. + :site-desc "" ; Used for RSS. + :enable-rss? true ; If true, creates a feed.xml in _site. + :ignored-dirs ["priv"] ; Directories to ignore org files in. + :layouts {} ; layouts loaded into memory + :partials {} ; partials loaded into memory :org-files []}) ; a list of org files, fetched when running setup. - -;; -- Default Config ----------------------------------------------------------- - - (defn make-dir-firn "make the _firn directory path." [dir-files] @@ -32,7 +32,7 @@ ([dir-files external-config] (let [base-config (merge starting-config external-config)] - (merge starting-config + (merge base-config {:dir-firn (make-dir-firn dir-files) :dir-data (str dir-files "/" (base-config :dir-data)) :dir-files dir-files diff --git a/clojure/src/firn/core.clj b/clojure/src/firn/core.clj index 9092f7ab..d330e5c6 100644 --- a/clojure/src/firn/core.clj +++ b/clojure/src/firn/core.clj @@ -43,7 +43,7 @@ [] ;; An option with a required argument [["-p" "--port PORT" "Port number" - :default 3333 + :default 4000 :parse-fn #(Integer/parseInt %) :validate [#(< 0 % 0x10000) "Must be a number between 0 and 65536"]] ;; A boolean option defaulting to nil @@ -91,4 +91,3 @@ "serve" (server/serve options) "build" (build/all-files options) "new" (build/new-site {})))))) - diff --git a/clojure/src/firn/dirwatch.clj b/clojure/src/firn/dirwatch.clj index c7fcaa6c..58af1983 100644 --- a/clojure/src/firn/dirwatch.clj +++ b/clojure/src/firn/dirwatch.clj @@ -12,7 +12,7 @@ (ns ^{:doc "Directory watcher" :author "Malcolm Sparks" :requires "JDK7"} - firn.dirwatch + firn.dirwatch (:import (java.io File) (java.nio.file FileSystems Path StandardWatchEventKinds WatchEvent WatchKey WatchService) (java.util.concurrent Executors ThreadFactory TimeUnit))) @@ -23,11 +23,11 @@ (defonce pool (Executors/newCachedThreadPool - (reify ThreadFactory - (newThread [_ runnable] - (doto (Thread. runnable) - (.setName (str "dirwatch-pool-" (swap! pool-counter inc))) - (.setDaemon true)))))) + (reify ThreadFactory + (newThread [_ runnable] + (doto (Thread. runnable) + (.setName (str "dirwatch-pool-" (swap! pool-counter inc))) + (.setDaemon true)))))) (defn ^:private register-path "Register a watch service with a filesystem path. @@ -41,9 +41,9 @@ StandardWatchEventKinds/ENTRY_MODIFY])) (doseq [^File dir (.. path toAbsolutePath toFile listFiles)] (when (. dir isDirectory) - (register-path ws (. dir toPath) event-atom)) + (register-path ws (. dir toPath) event-atom)) (when event-atom - (swap! event-atom conj {:file dir, :count 1, :action :create})))) + (swap! event-atom conj {:file dir, :count 1, :action :create})))) (defn ^:private wait-for-events [^WatchService ws f] (when ws ;; nil when this watcher is closed @@ -53,7 +53,7 @@ after the watcher is closed."))] (when (and k (.isValid k)) (doseq [^WatchEvent ev (.pollEvents k) :when (not= (.kind ev) - StandardWatchEventKinds/OVERFLOW)] + StandardWatchEventKinds/OVERFLOW)] (let [file (.toFile (.resolve ^Path (cast Path (.watchable k)) ^Path (cast Path (.context ev))))] (f {:file file :count (.count ev) @@ -110,5 +110,5 @@ [watcher] {:pre [(::watcher (meta watcher))]} (send-via pool watcher (fn [^WatchService w] - (when w (.close w)) - nil))) + (when w (.close w)) + nil))) diff --git a/clojure/src/firn/file.clj b/clojure/src/firn/file.clj index 90a7ba3d..2d9df293 100644 --- a/clojure/src/firn/file.clj +++ b/clojure/src/firn/file.clj @@ -5,6 +5,7 @@ You can view the file data-structure as it is made by the `make` function." (:require [cheshire.core :as json] + [clj-rss.core :as rss] [clojure.java.io :as io] [clojure.string :as s] [firn.layout :as layout] @@ -119,17 +120,17 @@ new-res [(+ acc-hours hour) (+ acc-minutes min)]] new-res))] (->> logbook - (reduce reduce-fn hours-minutes) - (u/timevec->time-str)))) + (reduce reduce-fn hours-minutes) + (u/timevec->time-str)))) (defn- sort-logbook "Loops over all logbooks, adds start and end unix timestamps." [logbook file] (let [mf #(org/parsed-org-date->unix-time %1 file)] (->> logbook - ;; Filter out timestamps if they don't have a start or end. + ;; Filter out timestamps if they don't have a start or end. (filter #(and (% :start) (% :end) (% :duration))) - ;; adds a unix timestamp for the :start and :end time so that's sortable. + ;; adds a unix timestamp for the :start and :end time so that's sortable. (map #(assoc % :start-ts (mf (:start %)) :end-ts (mf (:end %)))) (sort-by :start-ts #(> %1 %2))))) @@ -223,21 +224,47 @@ final (assoc config-with-data :processed-files with-html)] final) + ;; Otherwise continue... (let [next-file (first org-files) processed-file (process-one config next-file) + is-private (is-private? config processed-file) org-files (rest org-files) - output (assoc output (processed-file :path-web) processed-file) + output (if is-private + output + (assoc output (processed-file :path-web) processed-file)) keyword-map (keywords->map processed-file) new-site-map (merge keyword-map {:path (processed-file :path-web)})] ;; add to sitemap when file is not private. - (when-not (is-private? config processed-file) + (when-not is-private (swap! site-map conj new-site-map) (swap! site-links concat @site-links (-> processed-file :meta :links)) (swap! site-logs concat @site-logs (-> processed-file :meta :links))) ;; add links and logs to site wide data. (recur org-files output)))))) +(defn write-rss-file! + "Build an rss file. It sorts files by file:meta:date-created, writes to feed.xml" + [{:keys [processed-files site-url site-title dir-site site-desc] :as config}] + (println "Building rss file...") + (let [feed-file (str dir-site "feed.xml") + first-entry {:title site-title :link site-url :description site-desc} + make-rss (fn [[_ f]] + (hash-map :title (-> f :meta :title) + :link (str site-url "/" (-> f :path-web)) + :pubDate (u/org-date->java-date (-> f :meta :date-created)) + :description (str (f :as-html))))] + (io/make-parents feed-file) + (->> processed-files + (filter (fn [[_ f]] (-> f :meta :date-created))) + (map make-rss) + (sort-by :pubDate) + (reverse) + (u/prepend-vec first-entry) ; first entry must be about the site + (apply rss/channel-xml) + (spit feed-file))) + config) + (defn reload-requested-file "Take a request to a file, pulls the file out of memory grabs the path of the original file, reslurps it and reprocesses" @@ -245,3 +272,4 @@ (let [re-slurped (-> file :path io/file) re-processed (process-one config re-slurped)] re-processed)) + diff --git a/clojure/src/firn/layout.clj b/clojure/src/firn/layout.clj index 04f1f86c..a3c9ccff 100644 --- a/clojure/src/firn/layout.clj +++ b/clojure/src/firn/layout.clj @@ -53,6 +53,8 @@ ;; render a headline title. + + (and is-headline? (= opts :title)) (let [hl (org/get-headline org-tree action)] (-> hl :children first markup/to-html)) @@ -104,7 +106,6 @@ :date-updated (-> file :meta :date-updated) :date-created (-> file :meta :date-created)}) - (defn apply-layout "If a file has a template, render the file with it, or use the default layout" [config file layout] diff --git a/clojure/src/firn/markup.clj b/clojure/src/firn/markup.clj index 98a6bc97..725cc784 100644 --- a/clojure/src/firn/markup.clj +++ b/clojure/src/firn/markup.clj @@ -37,9 +37,9 @@ "converts `::*My Heading` => #my-heading" [anchor] (str "#" (-> anchor - (s/replace #"::\*" "") - (s/replace #" " "-") - (s/lower-case)))) + (s/replace #"::\*" "") + (s/replace #" " "-") + (s/lower-case)))) (defn internal-link-handler "Takes an org link and converts it into an html path." @@ -63,8 +63,8 @@ img-http-regex #"(http:\/\/|https:\/\/)(.*)\.(jpg|JPG|gif|GIF|png)" img-rel-regex #"(\.(.*))\.(jpg|JPG|gif|GIF|png)" img-make-url #(->> (re-matches img-file-regex link-href) - (take-last 2) - (s/join ".")) + (take-last 2) + (s/join ".")) ;; file regexs / ctor fns org-file-regex #"(file:)(.*)\.(org)(\:\:\*.+)?" http-link-regex #"https?:\/\/(?![^\" ]*(?:jpg|png|gif))[^\" ]+"] @@ -107,8 +107,7 @@ parent {:type "headline" :level level :children [v]} heading-priority (u/str->keywrd "span.firn-headline-priority.firn-headline-priority__" priority) heading-keyword (u/str->keywrd "span.firn-headline-keyword.firn-headline-keyword__" keywrd) - heading-anchor (-> parent org/get-headline-helper clean-anchor) - ;; _ (prn "heading anchor is " heading-anchor) + heading-anchor (-> parent org/get-headline-helper clean-anchor) heading-id+class #(u/str->keywrd "h" % heading-anchor ".firn-headline.firn-headline-" %) h-level (case level 1 (heading-id+class 1) @@ -155,7 +154,7 @@ (defn to-html "Recursively Parses the org-edn into hiccup. Some values don't get parsed (drawers) - yet. They return empty strings. - Don't destructure! - it can create uneven maps from possible nil vals on `V`" + Don't destructure! - with recursion, it can create uneven maps from possible nil vals on `v`" [v] (let [type (get v :type) children (get v :children) diff --git a/clojure/src/firn/org.clj b/clojure/src/firn/org.clj index 0dd398ce..c186d605 100644 --- a/clojure/src/firn/org.clj +++ b/clojure/src/firn/org.clj @@ -55,9 +55,9 @@ "Fetches a headline from an org-mode tree." [tree name] (->> (tree-seq map? :children tree) - (filter #(and (= "headline" (:type %)) - (= name (get-headline-helper %)))) - (first))) + (filter #(and (= "headline" (:type %)) + (= name (get-headline-helper %)))) + (first))) (defn get-headline-content "Same as get-headline, but removes the first child :title)." diff --git a/clojure/src/firn/server.clj b/clojure/src/firn/server.clj index ccb7747c..2f992eaf 100644 --- a/clojure/src/firn/server.clj +++ b/clojure/src/firn/server.clj @@ -87,7 +87,7 @@ dir-site-static dir-data dir-site-data]} @config!] - (prn "Reloading files..." file-path) + (println "Reloading files..." file-path) (cond (match-dir-and-action dir-partials :modxcreate) @@ -122,10 +122,10 @@ :start (let [{:keys [dir port] :or {dir (u/get-cwd) - port 3333}} (mount/args) + port 4000}} (mount/args) path-to-site (str dir "/_firn/_site") ;; NOTE: consider making this global, and so available to a sci repl? - config! (atom (-> dir config/prepare build/setup file/process-all)) + config! (atom (-> (mount/args) build/all-files)) {:keys [dir-layouts dir-partials dir-static dir-data]} @config! watch-list (map io/file [dir-layouts dir-partials dir-static dir-data])] @@ -135,7 +135,7 @@ (println "Building site...") (if-not (fs/exists? path-to-site) (println "Couldn't find a _firn/ folder. Have you run `Firn new` and created a site yet?") - (do (println "šŸ” Starting Firn development server on:" port) + (do (println "\nšŸ” Starting Firn development server on:" (str "http://localhost:" port)) (http/run-server (handler config!) {:port port})))) :stop diff --git a/clojure/src/firn/util.clj b/clojure/src/firn/util.clj index a9445a71..6151f913 100644 --- a/clojure/src/firn/util.clj +++ b/clojure/src/firn/util.clj @@ -5,8 +5,6 @@ (:import (java.lang Integer) (java.time LocalDate))) - - (set! *warn-on-reflection* true) ;; Some of these are borrowed from me.raynes.fs because I need to add ;; type hints for GraalVM @@ -80,9 +78,13 @@ (s/replace s #"_" "-")) ([s key-it?] (-> s - (s/replace #"_" "-") - (s/replace #" " "-") - (keyword)))) + (s/replace #"_" "-") + (s/replace #" " "-") + (keyword)))) + +(defn prepend-vec + [item vector] + (vec (cons item vector))) (defn io-file->keyword "Turn a filename into a keyword." @@ -109,7 +111,6 @@ eval-file (-> file-path slurp sci/eval-string)] eval-file)) - (defn load-fns-into-map "Takes a list of files and returns a map of filenames as :keywords -> file NOTE: It also EVALS (using sci) the files so they are in memory functions! @@ -157,6 +158,19 @@ ;; For interception thread macros and enabling printing the passed in value. (def spy #(do (println "DEBUG:" %) %)) +(defn java-date->unix-ts + [date] + (int (/ (.getTime ^java.util.Date date) 1000))) + +(defn org-date->java-date + "Converts <2020-02-25 05:51> -> java..." + [org-date] + (let [parse-fmt (java.text.SimpleDateFormat. "yyyy-MM-dd") + parse-fn (fn [s] (.parse ^java.text.SimpleDateFormat parse-fmt s))] + (-> org-date + (s/replace #"<" "") + (s/replace #">" "") + (parse-fn)))) (defn native-image? "Check if we are in the native-image or REPL." @@ -169,7 +183,6 @@ [& args] (keyword (apply str args))) - ;; Time ------------------------------------------------------------------------ ;; NOTE: timestr->hours-min + timevec->time-str could use better input testing? ;; At the very least, `Integer.` is an opportunity for errors when parsing. @@ -198,7 +211,6 @@ (double (/ (int (* 100 (/ (timestr->minutes tstr) 60))) 100))) - (defn timevec->time-str "Converts a vector of hours and minutes into readable time string. `[3 94]` > `4:34`" @@ -223,13 +235,12 @@ [date] (.toString ^java.time.LocalDate date)) - (defn date-range "Creates a range of dates between date A and date B. (date-range [])" [[sy sm sd] [ey em ed]] - (let [ s-date (date-make sy sm sd) - e-date (date-make ey em ed)] + (let [s-date (date-make sy sm sd) + e-date (date-make ey em ed)] (loop [curr-day s-date range [s-date]] (if (= curr-day e-date) diff --git a/clojure/test/firn/demo_org/file-footnotes.org b/clojure/test/firn/demo_org/file-footnotes.org index 26b7b4d2..dbe234bc 100644 --- a/clojure/test/firn/demo_org/file-footnotes.org +++ b/clojure/test/firn/demo_org/file-footnotes.org @@ -15,8 +15,8 @@ what about some context [fn:foo] * Footnotes -[fn:1] This test might not parse with the local bin. - +[fn:1] +This test might not parse with the local bin. [fn:foo] diff --git a/clojure/test/firn/layout_test.clj b/clojure/test/firn/layout_test.clj index 156307db..4d02ef46 100644 --- a/clojure/test/firn/layout_test.clj +++ b/clojure/test/firn/layout_test.clj @@ -11,7 +11,6 @@ res (sut/prepare sample-config test-file)] (t/is (every? #(contains? res %) [:render :title :site-map :site-links :site-logs :meta :partials :config])))) - (t/deftest get-layout (t/testing "The tf-layout file returns a sci function.") (let [test-file (stub/gtf :tf-layout :processed) diff --git a/clojure/test/firn/markup_test.clj b/clojure/test/firn/markup_test.clj index 5232d7a3..f904aa82 100644 --- a/clojure/test/firn/markup_test.clj +++ b/clojure/test/firn/markup_test.clj @@ -30,7 +30,6 @@ [:span.firn-img-with-caption [:img {:src "https://www.fillmurray.com/g/200/300.jpg"}] [:span.firn-img-caption "Fill murray"]]))) - (t/testing "img-link" (t/is (= (sut/link->html (sample-links :img-file)) diff --git a/clojure/test/firn/org_test.clj b/clojure/test/firn/org_test.clj index 89177a04..10d995c2 100644 --- a/clojure/test/firn/org_test.clj +++ b/clojure/test/firn/org_test.clj @@ -44,7 +44,6 @@ (t/is (= res1 "Image Tests")) (t/is (= res2 "Headlines"))))) - (t/deftest parsed-org-date->unix-time (t/testing "returns the expected value." (t/is (= 1585683360000 diff --git a/clojure/test/firn/util_test.clj b/clojure/test/firn/util_test.clj index beccf3ab..34489cae 100644 --- a/clojure/test/firn/util_test.clj +++ b/clojure/test/firn/util_test.clj @@ -64,7 +64,7 @@ (t/deftest find-index-of (t/testing "returns expected output." - (let [test-seq1 [ 1 2 3 4 5 6 7] + (let [test-seq1 [1 2 3 4 5 6 7] res1 (sut/find-index-of #(= % 3) test-seq1) test-seq2 [{:foo "bar" :baz 30} {:foo "non" :baz 10 :x "30"} {:baz 60}] res2 (sut/find-index-of #(> (% :baz) 20) test-seq2)] @@ -78,6 +78,11 @@ (t/is (= res1 "foo-bar")) (t/is (= res2 :foo-bar-foo))))) +(t/deftest prepend-vec + (t/testing "expected output." + (let [res (sut/prepend-vec 1 [2 3])] + (t/is (= res [1 2 3]))))) + ;; -- Time / Date Tests -------------------------------------------------------- (t/deftest timestr->hours-min