Improve utils (#341)

Author: rokasramas

utils/Makefile

@@ -1,5 +1,8 @@
 print-document-plan:
 	clojure -M:dp print-plan $(dp)
 
+export-generated-text:
+	clojure -M:generate $(dp) $(data_file) $(output_path) $(language)
+
 export-all-document-plans:
 	clojure -M:dp export-plans $(dir)

utils/src/utils/config.clj

@@ -1,6 +1,6 @@
 (ns utils.config
   (:require [mount.core :refer [defstate]]))
 
-(defn load-config [] {:graphql-url (or (System/getenv "GRAPHQL_URL") "http://localhost:3001/_graphql")})
+(defn load-config [] {:api-url (or (System/getenv "API_URL") "http://0.0.0.0:3001")})
 
 (defstate config :start (load-config))

utils/src/utils/document_plan.clj

@@ -6,7 +6,6 @@
             [clojure.tools.logging :as log]
             [jsonista.core :as json]
             [mount.core :as mount]
-            [org.httpkit.client :as http]
             [utils.config :refer [config]]
             [utils.queries :as queries]))
 
@@ -32,15 +31,9 @@
     (spit (format "%s/%s.json" dir id)
           (json/write-value-as-string document-plan write-mapper))))
 
-(defn run-query [url q]
-  @(http/post url {:headers {"Content-Type" "application/json"}
-                   :body    (->> q
-                                 :graphql
-                                 (json/write-value-as-string))}))
-
 (defn pprint-document-plan [name]
-  (-> (run-query (:graphql-url config)
-                 (queries/export-document-plan-query {:name name}))
+  (-> (queries/run-query (format "%s/_graphql" (:api-url config))
+                         (queries/export-document-plan-query {:name name}))
       :body
       (json/read-value read-mapper)
       :data :documentPlan
@@ -49,8 +42,8 @@
       (println)))
 
 (defn export-all-document-plans [output-dir]
-  (let [{:keys [body error]} (run-query (:graphql-url config)
-                                        (queries/export-document-plans-query {}))]
+  (let [{:keys [body error]} (queries/run-query (format "%s/_graphql" (:api-url config))
+                                                (queries/export-document-plans-query {}))]
     (if error
       (log/errorf "Failed with the error: %s" error)
       (doseq [dp (-> (json/read-value body read-mapper)

utils/src/utils/generate.clj

@@ -4,7 +4,9 @@
             [clojure.string :as str]
             [clojure.tools.logging :as log]
             [jsonista.core :as json]
-            [org.httpkit.client :as http])
+            [org.httpkit.client :as http]
+            [utils.config :refer [config]]
+            [mount.core :as mount])
   (:import (java.io File)
            (java.util UUID)))
 
@@ -15,7 +17,7 @@
     (doall (csv/read-csv reader))))
 
 (defn- post-generate [body]
-  @(http/request {:url     "http://localhost:3001/nlg/_bulk/"
+  @(http/request {:url     (format "%s/nlg/_bulk/" (:api-url config))
                   :method  :post
                   :headers {"Content-Type" "application/json"}
                   :body    (json/write-value-as-string body)}
@@ -25,7 +27,7 @@
                      (log/error (.getMessage ^Throwable error))))))
 
 (defn- fetch-results [id]
-  @(http/request {:url (format "http://localhost:3001/nlg/%s?format=raw" id)}
+  @(http/request {:url (format "%s/nlg/%s?format=raw" (:api-url config) id)}
                  (fn [{:keys [status body error]}]
                    (cond
                      (not= status 200) (log/errorf "Failed to fetch result `%s` with status %d" id status)
@@ -48,12 +50,14 @@
   [document-plan language data]
   (log/infof "Generating text for %s data items" (count data))
   (let [ids      (take (count data) (repeatedly #(str (UUID/randomUUID))))
+        id->index (zipmap ids (range))
         id->data (zipmap ids data)]
     (->> {:documentPlanName document-plan
           :dataRows         id->data
           :readerFlagValues {language true}}
          (post-generate)
          (map get-results)
+         (sort-by (fn [[id _]] (get id->index id)))
          (map-indexed (fn [i [id variants]]
                         (log/infof "%.2f%% ready" (float (* 100 (/ (inc i) (count ids)))))
                         [(get id->data id) variants])))))
@@ -82,6 +86,7 @@
     (save-data-with-variants output-file results header)))
 
 (defn -main [& [document-plan data-path output-path language]]
+  (mount/start)
   (if (and document-plan data-path output-path)
     (data->text document-plan (io/file data-path) (io/file output-path) (if-not (str/blank? language) language "Eng"))
     (println "Usage: pass in four parameters: name of the document plan, path to a data file, path to an output file, and language code")))

utils/src/utils/queries.clj

@@ -1,10 +1,18 @@
 (ns utils.queries
   (:require [graphql-builder.core :as core]
-            [graphql-builder.parser :refer [defgraphql]]))
+            [graphql-builder.parser :refer [defgraphql]]
+            [jsonista.core :as json]
+            [org.httpkit.client :as http]))
 
 (defgraphql doc-plan "export-document-plan.graphql")
 (defgraphql doc-plans "export-document-plans.graphql")
 
+(defn run-query [url q]
+  @(http/post url {:headers {"Content-Type" "application/json"}
+                   :body    (->> q
+                                 :graphql
+                                 (json/write-value-as-string))}))
+
 (def export-document-plans-query
   (-> doc-plans core/query-map :query :export-document-plans))