From 4b4ea7c56f846277d0ab6b2d00de1e73a9c5ca6d Mon Sep 17 00:00:00 2001 From: Jim Clark Date: Fri, 5 Jul 2024 21:18:17 -0700 Subject: [PATCH] Try harder to auth the pulls for missing images --- prompts/bb.edn | 2 +- prompts/dev/clean_local_images.clj | 36 +++++++++++++ prompts/runbook.md | 2 + prompts/src/creds.clj | 28 +++++++++++ prompts/src/docker.clj | 81 ++++++++++++++++++++++++------ prompts/src/prompts.clj | 76 ++++++++++++++++++---------- 6 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 prompts/dev/clean_local_images.clj create mode 100644 prompts/src/creds.clj diff --git a/prompts/bb.edn b/prompts/bb.edn index e1305b5..0961a0d 100644 --- a/prompts/bb.edn +++ b/prompts/bb.edn @@ -1,4 +1,4 @@ -{:paths ["src"] +{:paths ["src" "dev"] :deps {markdown-clj/markdown-clj {:mvn/version "1.12.1"} pogonos/pogonos {:mvn/version "0.2.1"} dev.weavejester/medley {:mvn/version "1.8.0"} diff --git a/prompts/dev/clean_local_images.clj b/prompts/dev/clean_local_images.clj new file mode 100644 index 0000000..c447010 --- /dev/null +++ b/prompts/dev/clean_local_images.clj @@ -0,0 +1,36 @@ +(ns clean-local-images + (:require [docker] + [clojure.pprint :refer [pprint]])) + +(def images + #{"vonwig/prompts" + "vonwig/function_write_file" + "vonwig/docker_scout_tag_recommendation" + "vonwig/extractor-node" + "vonwig/go-linguist" + "vonwig/codescope" + "vonwig/pre-commit" + "markdownlint/markdownlint" + "hadolint/hadolint" + "docker/lsp"}) + +(comment + (docker/delete-image {:image "vonwig/function_write_files"})) + +(defn repo? [images] + (fn [tag-or-digest] + (some (fn [image] (.startsWith tag-or-digest image)) images))) + +(defn -main [] + (->> (docker/images {}) + #_(map #(concat (:RepoTags %) (:RepoDigests %))) + (filter #(some (repo? images) (concat (:RepoTags %) (:RepoDigests %)))) + (map :Id) + (map #(docker/delete-image {:image %})))) + +(-main) + +(comment + (pprint (docker/images {})) + ) + diff --git a/prompts/runbook.md b/prompts/runbook.md index bb5c8c3..aa0df2e 100644 --- a/prompts/runbook.md +++ b/prompts/runbook.md @@ -2,6 +2,8 @@ ## Running the docker prompts +### Directly + ```sh #docker:command=run-docker bb -m prompts /Users/slim/docker/labs-make-runbook jimclark106 darwin docker diff --git a/prompts/src/creds.clj b/prompts/src/creds.clj new file mode 100644 index 0000000..81fdf0f --- /dev/null +++ b/prompts/src/creds.clj @@ -0,0 +1,28 @@ +(ns creds + (:require [babashka.fs :as fs] + [babashka.process :as process] + [cheshire.core :as json])) + +(defn credential-helper + "doesn't work without a HOME and docker desktop install" + [key] + (let [path (fs/file (.get (System/getenv) "HOME") ".docker" "config.json")] + (when (.exists path) + (when-let [cred-store (-> (slurp path) (json/parse-string keyword) :credsStore)] + (try + (-> + (process/process [(format "docker-credential-%s" cred-store) "get"] + {:out :string :in key}) + (deref) + :out + (json/parse-string keyword)) + (catch Throwable _)))))) + +(defn credential-helper->jwt + "doesn't work without a HOME and docker desktop install" + [] + (:Secret (credential-helper "https://index.docker.io/v1//access-token"))) + +(comment + (credential-helper->jwt)) + diff --git a/prompts/src/docker.clj b/prompts/src/docker.clj index 1acedfe..9163b8a 100644 --- a/prompts/src/docker.clj +++ b/prompts/src/docker.clj @@ -2,19 +2,51 @@ (:require [babashka.curl :as curl] [cheshire.core :as json] - [clojure.pprint :refer [pprint]]) + [clojure.pprint :refer [pprint]] + [clojure.spec.alpha :as spec] + [creds]) (:import [java.util Base64])) (defn encode [to-encode] (.encodeToString (Base64/getEncoder) (.getBytes to-encode))) -(defn pull-image [{:keys [image identity-token]}] +;; https://index.docker.io/v1/ does not return IdentityTokens so we +;; probably won't use this endpoint +#_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} +(defn auth + [creds] + (curl/post + "http://localhost/auth" + {:raw-args ["--unix-socket" "/var/run/docker.sock"] + :body (json/generate-string creds)})) + +(defn pull-image [{:keys [image identity-token creds]}] (curl/post (format "http://localhost/images/create?fromImage=%s" image) {:raw-args ["--unix-socket" "/var/run/docker.sock"] :throw false - :headers {"X-Registry-Auth" (encode (json/generate-string {:identity-token identity-token}))}})) + :headers {"X-Registry-Auth" + ;; I don't think we'll be pulling images + ;; from registries that support identity tokens + (-> (cond + identity-token {:identitytoken identity-token} + creds creds) + (json/generate-string) + (encode))}})) + +(defn list-images [m] + (curl/get + "http://localhost/images/json" + {:raw-args ["--unix-socket" "/var/run/docker.sock"] + :query-params {:filters (json/generate-string m)} + :throw false})) + +(defn delete-image [{:keys [image]}] + (curl/delete + (format "http://localhost/images/%s?force=true" image) + {:raw-args ["--unix-socket" "/var/run/docker.sock"] + :throw false})) (defn container->archive [{:keys [Id path]}] (curl/get @@ -45,10 +77,9 @@ (defn inspect-container [{:keys [Id]}] (curl/get - (format "http://localhost/containers/%s/json" Id) - {:raw-args ["--unix-socket" "/var/run/docker.sock"] - :throw false - })) + (format "http://localhost/containers/%s/json" Id) + {:raw-args ["--unix-socket" "/var/run/docker.sock"] + :throw false})) (defn start-container [{:keys [Id]}] (curl/post @@ -95,6 +126,7 @@ (def delete (comp (status? 204 "delete-container") delete-container)) (def get-archive (comp (status? 200 "container->archive") container->archive)) (def pull (comp (status? 200 "pull-image") pull-image)) +(def images (comp ->json list-images)) (def sample {:image "docker/lsp:latest" :entrypoint "/app/result/bin/docker-lsp" @@ -102,9 +134,21 @@ "--vs-machine-id" "none" "--workspace" "/docker"]}) +(spec/def ::host-dir string?) +(spec/def ::entrypoint string?) +(spec/def ::user string?) +(spec/def ::pat string?) +(spec/def ::image string?) +(spec/def ::command (spec/coll-of string?)) +(spec/def ::container-definition (spec/keys :opt-un [::host-dir ::entrypoint ::command ::user ::pat] + :req-un [::image])) + +;; TODO verify that m is a container-definition (defn run-function [m] - (when (:identity-token m) - (pull m)) + (when (and (:user m) (or (:pat m) (creds/credential-helper->jwt))) + (pull (assoc m :creds {:username (:user m) + :password (or (:pat m) (creds/credential-helper->jwt)) + :serveraddress "https://index.docker.io/v1/"}))) (let [x (create m)] (start x) (wait x) @@ -119,12 +163,21 @@ (def extract-facts run-function) (comment - (pprint (json/parse-string (extract-facts (assoc sample :host-dir "/Users/slim/docker/genai-stack")) keyword)) - (extract-facts {:image "vonwig/go-linguist:latest" :command ["-json"] :host-dir "/Users/slim/docker/labs-make-runbook"}) + (pprint + (json/parse-string + (extract-facts + (assoc sample + :host-dir "/Users/slim/docker/genai-stack" + :user "jimclark106")) keyword)) + (docker/delete-image {:image "vonwig/go-linguist:latest"}) + (extract-facts {:image "vonwig/go-linguist:latest" + :command ["-json"] + :host-dir "/Users/slim/docker/labs-make-runbook" + :user "jimclark106"}) (pprint (json/parse-string - (extract-facts - {:image "vonwig/extractor-node:latest" - :host-dir "/Users/slim/docker/labs-make-runbook"}) + (extract-facts + {:image "vonwig/extractor-node:latest" + :host-dir "/Users/slim/docker/labs-make-runbook"}) keyword))) diff --git a/prompts/src/prompts.clj b/prompts/src/prompts.clj index d3c9d32..d742e32 100644 --- a/prompts/src/prompts.clj +++ b/prompts/src/prompts.clj @@ -5,20 +5,23 @@ [clojure.core.async :as async] [clojure.edn :as edn] [clojure.java.io :as io] + [clojure.pprint :refer [pprint]] [clojure.string :as string] [clojure.tools.cli :as cli] + creds [docker] [git] + [jsonrpc] [markdown.core :as markdown] [medley.core :as medley] [openai] - [jsonrpc] - [clojure.pprint :refer [pprint]] [pogonos.core :as stache] [pogonos.partials :as partials] [selmer.parser :as selmer])) -(defn- facts [project-facts user platform] +(defn- facts + "fix up facts before sending to templates" + [project-facts user platform] (medley/deep-merge {:platform platform :username user @@ -63,14 +66,15 @@ "reduces into m using a container function params dir - the host dir that the container will mount read-only at /project - identity-token - a DockerHub identity token m - the map to merge into container-definition - the definition for the function" - [dir identity-token m container-definition] + [dir m container-definition] (try (medley/deep-merge m - (let [{:keys [pty-output exit-code]} (docker/extract-facts (assoc container-definition :host-dir dir :identity-token identity-token))] + (let [{:keys [pty-output exit-code]} (docker/extract-facts + (-> container-definition + (assoc :host-dir dir)))] (when (= 0 exit-code) (case (:output-handler container-definition) "linguist" (->> (json/parse-string pty-output keyword) vals (into []) (assoc {} :linguist)) @@ -84,13 +88,15 @@ m))) (comment + ;; TODO move this to an assertion (facts (fact-reducer "/Users/slim/docker/labs-make-runbook" - "" {} {:image "vonwig/go-linguist:latest" :command ["-json"] - :output-handler "linguist"}) + :output-handler "linguist" + :user "jimclark106" + :pat (creds/credential-helper->jwt)}) "jimclark106" "darwin")) (defn collect-extractors [dir] @@ -113,14 +119,20 @@ (-> (markdown/parse-metadata (io/file dir "README.md")) first) :extractors :functions)) -(defn all-facts +(defn run-extractors "returns a map of extracted *math-context* params project-root - the host project root dir identity-token - a valid Docker login auth token - dir - a prompst directory with a valid README.md" - [project-root identity-token dir] - (reduce (partial fact-reducer project-root identity-token) {} (collect-extractors dir))) + dir - a prompts directory with a valid README.md" + [{:keys [host-dir prompts user pat]}] + (reduce + (partial fact-reducer host-dir) + {} + (->> (collect-extractors prompts) + (map (fn [m] (merge m + (when user {:user user}) + (when pat {:pat pat}))))))) ;; registry of prompts directories stores in the docker-prompts volumes (def registry-file "/prompts/registry.edn") @@ -150,10 +162,15 @@ "docker")) (defn get-prompts [& args] - (let [[project-root user platform dir & {:keys [identity-token]}] args + (let [[project-root user platform prompts-dir & {:keys [pat]}] args ;; TODO the docker default no longer makes sense here - prompt-dir (get-dir dir) - m (all-facts project-root identity-token prompt-dir) + prompt-dir (get-dir prompts-dir) + m (run-extractors + (merge {:host-dir project-root + :prompts prompt-dir + :user user + :platform platform} + (when pat {:pat pat}))) renderer (partial selma-render prompt-dir (facts m user platform)) prompts (->> (fs/list-dir prompt-dir) (filter (name-matches prompt-file-pattern)) @@ -161,7 +178,7 @@ (into []))] (map (comp merge-role renderer fs/file) prompts))) -(defn function-handler [{:keys [functions] :as opts} function-name json-arg-string {:keys [resolve fail]}] +(defn function-handler [{:keys [functions user pat] :as opts} function-name json-arg-string {:keys [resolve fail]}] (if-let [definition (-> (->> (filter #(= function-name (-> % :function :name)) functions) first) @@ -172,7 +189,9 @@ (let [function-call (merge (:container definition) (dissoc opts :functions) - {:command [json-arg-string]}) + {:command [json-arg-string]} + (when user {:user user}) + (when pat {:pat pat})) {:keys [pty-output exit-code]} (docker/run-function function-call)] (if (= 0 exit-code) (resolve pty-output) @@ -187,13 +206,18 @@ (defn- run-prompts [prompts & args] - (let [prompt-dir (get-dir (last args)) + (let [[host-dir user platform prompts-dir & {:keys [pat]}] args + prompt-dir (get-dir prompts-dir) m (collect-metadata prompt-dir) functions (collect-functions prompt-dir) [c h] (openai/chunk-handler (partial function-handler - {:functions functions - :host-dir (first (rest args))}))] + (merge + {:functions functions + :host-dir host-dir + :user user + :platform platform} + (when pat {:pat pat}))))] (openai/openai (merge @@ -205,7 +229,7 @@ (defn- conversation-loop [& args] (async/go-loop - [prompts (apply get-prompts (rest args))] + [prompts (apply get-prompts args)] (let [{:keys [messages finish-reason] :as m} (async/