diff --git a/deps.edn b/deps.edn index dbe0fb0..ce302d9 100644 --- a/deps.edn +++ b/deps.edn @@ -2,9 +2,10 @@ :deps {com.layerware/hugsql {:mvn/version "0.4.9"} org.xerial/sqlite-jdbc {:mvn/version "3.20.0"} org.clojure/java.jdbc {:mvn/version "0.7.0"} + tea-time {:mvn/version "1.0.0"} + ring/ring-core {:mvn/version "1.7.0-RC1"} ring/ring-jetty-adapter {:mvn/version "1.7.0-RC1"} - tea-time {:mvn/version "1.0.0"} metosin/reitit-core {:mvn/version "0.1.3"} metosin/reitit-ring {:mvn/version "0.1.3"} metosin/reitit-spec {:mvn/version "0.1.3"} diff --git a/src/cljdoc/clojars_stats.clj b/src/cljdoc/clojars_stats.clj index b806d13..65d1cd6 100644 --- a/src/cljdoc/clojars_stats.clj +++ b/src/cljdoc/clojars_stats.clj @@ -6,7 +6,8 @@ [clojure.string :as string] [clojure.java.io :as io] [tea-time.core :as tt]) - (:import (java.time Instant))) + (:import (java.time Instant Duration LocalDate) + (java.time.temporal ChronoUnit))) (defn stats-files [] (->> (slurp "https://clojars.org/stats/") @@ -14,20 +15,26 @@ (map #(str "https://clojars.org/stats/" %)) (set))) -(defn read-file [uri] - (let [data-dir (io/file "data")] - (if (.exists data-dir) - (let [local-file (io/file data-dir (last (string/split uri #"/")))] - (when-not (.exists local-file) - (println "Downloading" (.getName local-file)) - (spit local-file (slurp uri))) - (edn/read-string (slurp local-file))) - (edn/read-string (slurp uri))))) - (defn uri->date [uri] (some->> (re-find #"(\d{4})(\d{2})(\d{2})" uri) (rest) - (string/join "-"))) + (string/join "-") + (LocalDate/parse))) + +(defn stats-for-date-exist? [db-spec date] + (println "Checking" date) + (-> (sql/query db-spec ["select exists(SELECT date FROM stats where date = ? limit 1)" date]) + first first val pos?)) + +(defn to-import [db-spec] + (let [retention-years 1 + statsf (stats-files)] + (->> (zipmap (map uri->date statsf) statsf) + (remove #(.isBefore (key %) + (.minusYears (LocalDate/now) retention-years))) + (remove #(stats-for-date-exist? db-spec (key %))) + vals + sort))) (defn parse-file [uri] (let [date (uri->date uri)] @@ -39,7 +46,7 @@ :artifact_id (do (assert artifact) artifact) :version (do (assert (key d)) (key d)) :downloads (do (assert (val d)) (val d))})) - (read-file uri)))) + (edn/read-string (slurp uri))))) (defn import-file! [db f] (printf "Reading %s ..." f) @@ -65,20 +72,20 @@ (import! db (take 20 todo)) (< 0 (count todo)))) +(defn start! [{:keys [db-spec]}] + (db/create-stats-table db-spec) + (db/create-imported-files-table db-spec) + (println "Starting background process...") + (tt/start!) + (tt/every! (* 1 60) (bound-fn [] (update! db-spec)))) + (defn -main [db-file] - (let [db {:classname "org.sqlite.JDBC" - :subprotocol "sqlite" - :subname db-file}] - (when-not (.exists (java.io.File. db-file)) - (println "Creating tables...") - (db/create-stats-table db) - (db/create-imported-files-table db)) - (println "Starting background process...") - (tt/start!) - (api/start! {:db db :port 3000}) - (tt/every! (* 1 60) (bound-fn [] (update! db))) - (let [signal (java.util.concurrent.CountDownLatch. 1)] - (.await signal)))) + (start! {:db-spec {:classname "org.sqlite.JDBC" + :subprotocol "sqlite" + :subname db-file} + :retention (Duration/of 365 ChronoUnit/DAYS)}) + (let [signal (java.util.concurrent.CountDownLatch. 1)] + (.await signal))) (comment (def db {:classname "org.sqlite.JDBC" @@ -86,6 +93,11 @@ :subname "clojars-stats.db"}) (time (-main)) + (.minusYears (LocalDate/now) 1) + + (to-import db) + + (stats-for-date-exist? db (LocalDate/parse "2017-05-01")) (db/create-imported-files-table db) (db/create-stats-table db)