Try harder to auth the pulls for missing images

prompts/bb.edn

@@ -1,4 +1,4 @@
-{:paths ["src"]
+{:paths ["src" "dev"]
  :deps {markdown-clj/markdown-clj {:mvn/version "1.12.1"}
         pogonos/pogonos {:mvn/version "0.2.1"}
         dev.weavejester/medley {:mvn/version "1.8.0"}

prompts/dev/clean_local_images.clj

@@ -0,0 +1,36 @@
+(ns clean-local-images
+  (:require [docker]
+            [clojure.pprint :refer [pprint]]))
+
+(def images
+  #{"vonwig/prompts"
+    "vonwig/function_write_file"
+    "vonwig/docker_scout_tag_recommendation"
+    "vonwig/extractor-node"
+    "vonwig/go-linguist"
+    "vonwig/codescope"
+    "vonwig/pre-commit"
+    "markdownlint/markdownlint"
+    "hadolint/hadolint"
+    "docker/lsp"})
+
+(comment
+  (docker/delete-image {:image "vonwig/function_write_files"}))
+
+(defn repo? [images]
+  (fn [tag-or-digest]
+    (some (fn [image] (.startsWith tag-or-digest image)) images)))
+
+(defn -main []
+  (->> (docker/images {})
+       #_(map #(concat (:RepoTags %) (:RepoDigests %)))
+       (filter #(some (repo? images) (concat (:RepoTags %) (:RepoDigests %))))
+       (map :Id)
+       (map #(docker/delete-image {:image %}))))
+
+(-main)
+
+(comment
+  (pprint (docker/images {}))
+  )
+

prompts/runbook.md

@@ -2,6 +2,8 @@
 
 ## Running the docker prompts
 
+### Directly
+
 ```sh
 #docker:command=run-docker
 bb -m prompts /Users/slim/docker/labs-make-runbook jimclark106 darwin docker

prompts/src/creds.clj

@@ -0,0 +1,28 @@
+(ns creds
+  (:require [babashka.fs :as fs]
+            [babashka.process :as process]
+            [cheshire.core :as json]))
+
+(defn credential-helper 
+  "doesn't work without a HOME and docker desktop install"
+  [key]
+  (let [path (fs/file (.get (System/getenv) "HOME") ".docker" "config.json")]
+    (when (.exists path)
+      (when-let [cred-store (-> (slurp path) (json/parse-string keyword) :credsStore)]
+        (try
+          (->
+            (process/process [(format "docker-credential-%s" cred-store) "get"]
+                             {:out :string :in key})
+            (deref)
+            :out
+            (json/parse-string keyword))
+          (catch Throwable _))))))
+
+(defn credential-helper->jwt 
+  "doesn't work without a HOME and docker desktop install"
+  []
+  (:Secret (credential-helper "https://index.docker.io/v1//access-token")))
+
+(comment
+  (credential-helper->jwt))
+

prompts/src/docker.clj

@@ -2,19 +2,51 @@
   (:require
    [babashka.curl :as curl]
    [cheshire.core :as json]
-   [clojure.pprint :refer [pprint]]) 
+   [clojure.pprint :refer [pprint]]
+   [clojure.spec.alpha :as spec]
+   [creds])
   (:import
    [java.util Base64]))
 
 (defn encode [to-encode]
   (.encodeToString (Base64/getEncoder) (.getBytes to-encode)))
 
-(defn pull-image [{:keys [image identity-token]}]
+;; https://index.docker.io/v1/ does not return IdentityTokens so we 
+;; probably won't use this endpoint
+#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]}
+(defn auth
+  [creds]
+  (curl/post
+   "http://localhost/auth"
+   {:raw-args ["--unix-socket" "/var/run/docker.sock"]
+    :body (json/generate-string creds)}))
+
+(defn pull-image [{:keys [image identity-token creds]}]
   (curl/post
    (format "http://localhost/images/create?fromImage=%s" image)
    {:raw-args ["--unix-socket" "/var/run/docker.sock"]
     :throw false
-    :headers {"X-Registry-Auth" (encode (json/generate-string {:identity-token identity-token}))}}))
+    :headers {"X-Registry-Auth"
+              ;; I don't think we'll be pulling images
+              ;; from registries that support identity tokens
+              (-> (cond
+                    identity-token {:identitytoken identity-token}
+                    creds creds)
+                  (json/generate-string)
+                  (encode))}}))
+
+(defn list-images [m]
+  (curl/get
+   "http://localhost/images/json"
+   {:raw-args ["--unix-socket" "/var/run/docker.sock"]
+    :query-params {:filters (json/generate-string m)}
+    :throw false}))
+
+(defn delete-image [{:keys [image]}]
+  (curl/delete
+   (format "http://localhost/images/%s?force=true" image)
+   {:raw-args ["--unix-socket" "/var/run/docker.sock"]
+    :throw false}))
 
 (defn container->archive [{:keys [Id path]}]
   (curl/get
@@ -45,10 +77,9 @@
 
 (defn inspect-container [{:keys [Id]}]
   (curl/get
-    (format "http://localhost/containers/%s/json" Id)
-    {:raw-args ["--unix-socket" "/var/run/docker.sock"]
-     :throw false
-     }))
+   (format "http://localhost/containers/%s/json" Id)
+   {:raw-args ["--unix-socket" "/var/run/docker.sock"]
+    :throw false}))
 
 (defn start-container [{:keys [Id]}]
   (curl/post
@@ -95,16 +126,29 @@
 (def delete (comp (status? 204 "delete-container") delete-container))
 (def get-archive (comp (status? 200 "container->archive") container->archive))
 (def pull (comp (status? 200 "pull-image") pull-image))
+(def images (comp ->json list-images))
 
 (def sample {:image "docker/lsp:latest"
              :entrypoint "/app/result/bin/docker-lsp"
              :command ["project-facts"
                        "--vs-machine-id" "none"
                        "--workspace" "/docker"]})
 
+(spec/def ::host-dir string?)
+(spec/def ::entrypoint string?)
+(spec/def ::user string?)
+(spec/def ::pat string?)
+(spec/def ::image string?)
+(spec/def ::command (spec/coll-of string?))
+(spec/def ::container-definition (spec/keys :opt-un [::host-dir ::entrypoint ::command ::user ::pat]
+                                           :req-un [::image]))
+
+;; TODO verify that m is a container-definition
 (defn run-function [m]
-  (when (:identity-token m)
-    (pull m))
+  (when (and (:user m) (or (:pat m) (creds/credential-helper->jwt))) 
+    (pull (assoc m :creds {:username (:user m)
+                           :password (or (:pat m) (creds/credential-helper->jwt))
+                           :serveraddress "https://index.docker.io/v1/"})))
   (let [x (create m)]
     (start x)
     (wait x)
@@ -119,12 +163,21 @@
 (def extract-facts run-function)
 
 (comment
-  (pprint (json/parse-string (extract-facts (assoc sample :host-dir "/Users/slim/docker/genai-stack")) keyword))
-  (extract-facts {:image "vonwig/go-linguist:latest" :command ["-json"] :host-dir "/Users/slim/docker/labs-make-runbook"})
+  (pprint 
+    (json/parse-string 
+      (extract-facts 
+        (assoc sample 
+               :host-dir "/Users/slim/docker/genai-stack"
+               :user "jimclark106")) keyword))
+  (docker/delete-image {:image "vonwig/go-linguist:latest"})
+  (extract-facts {:image "vonwig/go-linguist:latest" 
+                  :command ["-json"] 
+                  :host-dir "/Users/slim/docker/labs-make-runbook"
+                  :user "jimclark106"})
   (pprint
    (json/parse-string
-     (extract-facts
-       {:image "vonwig/extractor-node:latest"
-        :host-dir "/Users/slim/docker/labs-make-runbook"})
+    (extract-facts
+     {:image "vonwig/extractor-node:latest"
+      :host-dir "/Users/slim/docker/labs-make-runbook"})
     keyword)))
 

prompts/src/prompts.clj

@@ -5,20 +5,23 @@
    [clojure.core.async :as async]
    [clojure.edn :as edn]
    [clojure.java.io :as io]
+   [clojure.pprint :refer [pprint]]
    [clojure.string :as string]
    [clojure.tools.cli :as cli]
+   creds
    [docker]
    [git]
+   [jsonrpc]
    [markdown.core :as markdown]
    [medley.core :as medley]
    [openai]
-   [jsonrpc]
-   [clojure.pprint :refer [pprint]]
    [pogonos.core :as stache]
    [pogonos.partials :as partials]
    [selmer.parser :as selmer]))
 
-(defn- facts [project-facts user platform]
+(defn- facts
+  "fix up facts before sending to templates"
+  [project-facts user platform]
   (medley/deep-merge
    {:platform platform
     :username user
@@ -63,14 +66,15 @@
   "reduces into m using a container function
      params
        dir - the host dir that the container will mount read-only at /project
-       identity-token - a DockerHub identity token
        m - the map to merge into
        container-definition - the definition for the function"
-  [dir identity-token m container-definition]
+  [dir m container-definition]
   (try
     (medley/deep-merge
      m
-     (let [{:keys [pty-output exit-code]} (docker/extract-facts (assoc container-definition :host-dir dir :identity-token identity-token))]
+     (let [{:keys [pty-output exit-code]} (docker/extract-facts
+                                           (-> container-definition
+                                               (assoc :host-dir dir)))]
        (when (= 0 exit-code)
          (case (:output-handler container-definition)
            "linguist" (->> (json/parse-string pty-output keyword) vals (into []) (assoc {} :linguist))
@@ -84,13 +88,15 @@
       m)))
 
 (comment
+  ;; TODO move this to an assertion
   (facts
    (fact-reducer "/Users/slim/docker/labs-make-runbook"
-                 ""
                  {}
                  {:image "vonwig/go-linguist:latest"
                   :command ["-json"]
-                  :output-handler "linguist"})
+                  :output-handler "linguist"
+                  :user "jimclark106"
+                  :pat (creds/credential-helper->jwt)})
    "jimclark106" "darwin"))
 
 (defn collect-extractors [dir]
@@ -113,14 +119,20 @@
    (-> (markdown/parse-metadata (io/file dir "README.md")) first)
    :extractors :functions))
 
-(defn all-facts
+(defn run-extractors
   "returns a map of extracted *math-context*
      params
        project-root - the host project root dir
        identity-token - a valid Docker login auth token
-       dir - a prompst directory with a valid README.md"
-  [project-root identity-token dir]
-  (reduce (partial fact-reducer project-root identity-token) {} (collect-extractors dir)))
+       dir - a prompts directory with a valid README.md"
+  [{:keys [host-dir prompts user pat]}]
+  (reduce
+   (partial fact-reducer host-dir)
+   {}
+   (->> (collect-extractors prompts)
+        (map (fn [m] (merge m
+                            (when user {:user user})
+                            (when pat {:pat pat})))))))
 
 ;; registry of prompts directories stores in the docker-prompts volumes
 (def registry-file "/prompts/registry.edn")
@@ -150,18 +162,23 @@
    "docker"))
 
 (defn get-prompts [& args]
-  (let [[project-root user platform dir & {:keys [identity-token]}] args
+  (let [[project-root user platform prompts-dir & {:keys [pat]}] args
         ;; TODO the docker default no longer makes sense here
-        prompt-dir (get-dir dir)
-        m (all-facts project-root identity-token prompt-dir)
+        prompt-dir (get-dir prompts-dir)
+        m (run-extractors
+           (merge {:host-dir project-root
+                   :prompts prompt-dir
+                   :user user
+                   :platform platform}
+                  (when pat {:pat pat})))
         renderer (partial selma-render prompt-dir (facts m user platform))
         prompts (->> (fs/list-dir prompt-dir)
                      (filter (name-matches prompt-file-pattern))
                      (sort-by fs/file-name)
                      (into []))]
     (map (comp merge-role renderer fs/file) prompts)))
 
-(defn function-handler [{:keys [functions] :as opts} function-name json-arg-string {:keys [resolve fail]}]
+(defn function-handler [{:keys [functions user pat] :as opts} function-name json-arg-string {:keys [resolve fail]}]
   (if-let [definition (->
                        (->> (filter #(= function-name (-> % :function :name)) functions)
                             first)
@@ -172,7 +189,9 @@
         (let [function-call (merge
                              (:container definition)
                              (dissoc opts :functions)
-                             {:command [json-arg-string]})
+                             {:command [json-arg-string]}
+                             (when user {:user user})
+                             (when pat {:pat pat}))
               {:keys [pty-output exit-code]} (docker/run-function function-call)]
           (if (= 0 exit-code)
             (resolve pty-output)
@@ -187,13 +206,18 @@
 
 (defn- run-prompts
   [prompts & args]
-  (let [prompt-dir (get-dir (last args))
+  (let [[host-dir user platform prompts-dir & {:keys [pat]}] args
+        prompt-dir (get-dir prompts-dir)
         m (collect-metadata prompt-dir)
         functions (collect-functions prompt-dir)
         [c h] (openai/chunk-handler (partial
                                      function-handler
-                                     {:functions functions
-                                      :host-dir (first (rest args))}))]
+                                     (merge
+                                      {:functions functions
+                                       :host-dir host-dir
+                                       :user user
+                                       :platform platform}
+                                      (when pat {:pat pat}))))]
 
     (openai/openai
      (merge
@@ -205,7 +229,7 @@
 (defn- conversation-loop
   [& args]
   (async/go-loop
-   [prompts (apply get-prompts (rest args))]
+   [prompts (apply get-prompts args)]
     (let [{:keys [messages finish-reason] :as m} (async/<!! (apply run-prompts prompts args))]
       (if (= "tool_calls" finish-reason)
         (do
@@ -242,7 +266,7 @@
        (update-in m [:prompts] (fn [coll] (remove (fn [{:keys [type]}] (= type (second args))) coll)))))
 
     (= "run" (first args))
-    (async/<!! (apply conversation-loop args))
+    (async/<!! (apply conversation-loop (rest args)))
 
     :else
     (apply get-prompts args)))
@@ -251,8 +275,9 @@
   (println
    (json/generate-string (apply -run-command args))))
 
-(def cli-opts [[nil "--identity-token TOKEN" "Supply a Docker identity token for pulling images"]
-               [nil "--jsonrpc" "Output JSON-RPC notifications"]])
+(def cli-opts [[nil "--jsonrpc" "Output JSON-RPC notifications"]
+               [nil "--user USER" "The hub user"]
+               [nil "--pat PAT" "A hub PAT"]])
 
 (defn -main [& args]
   (try
@@ -266,8 +291,7 @@
                  jsonrpc/-println)))
       (apply prompts (concat
                       arguments
-                      (when (:identity-token options)
-                        [:identity-token (:identity-token options)]))))
+                      (when-let [pat (:pat options)] [:pat pat]))))
     (catch Throwable t
       (warn "Error: {{ exception }}" {:exception t})
       (System/exit 1))))