Verified Commit 3946a3a0 authored by Snorre Magnus Davøen's avatar Snorre Magnus Davøen 💬
Browse files

Release 0.2.0 with terminate-pipelines



New subcommand terminate-pipelines to handle deletion of old pipeline logs and
artifacts. Old terminate command renamed to terminate-tags.
Signed-off-by: Snorre Magnus Davøen's avatarSnorre Magnus Davøen <snorre.davoen@nsd.no>
parent ca49d336
Pipeline #27319 failed with stage
in 18 seconds
stages:
- locate
- terminate
locate:
image: docker.nsd.no/ops/containinator:0.1.0
stage: locate
script:
- ./containinator -Djava.library.path=/opt/java/openjdk/lib/ --gitlab https://gitlab.nsd.no locate --sort tag-count --limit 30 -f tag-count
......@@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. This change
## [Unreleased]
## [v0.2.0] - 2019-12-13
### Added
- `terminate-pipelines` command to terminate pipelines (including logs and artifacts)
### Changed
- Renamed original `terminate` command to `terminate-tags`
## [v0.1.0] - 2019-06-26
### Added
- Initial implementation of command line app
......@@ -10,4 +16,7 @@ All notable changes to this project will be documented in this file. This change
- terminate argument to instruct GitLab to bulk delete Docker image tags from projects' registries
- Docker image to bundle all required dependencies (dynamically linked libsunec and others)
[Unreleased]: https://gitlab.nsd.no/ops/containinator/compare/v0.1.0...master
[Unreleased]: https://gitlab.nsd.no/ops/containinator/compare/v0.2.0...master
[v0.2.0]: https://gitlab.nsd.no/ops/containinator/compare/v0.1.0...v0.2.0
[v0.1.0]: https://gitlab.nsd.no/ops/containinator/-/tags/v0.1.0
......@@ -53,7 +53,7 @@ The `terminate` command sends bulk tag deletion commands for all projects with a
## Examples
```bash
./containinator -Djava.library.path=$JAVA_HOME/lib/ --token <yourtoken> --gitlab https://gitlab.<yourdomain>.<tld> locate --sort tag-count --limit 20
./containinator -Djava.library.path=$JAVA_HOME/lib/ --token <yourtoken> --gitlab https://gitlab.<yourdomain>.<tld> locate --sort tag-count --limit 20 -f tag-count -f registry-count
Locating projects for given arguments on https://gitlab.nsd.no Please be patient, it could take some time.
......@@ -80,13 +80,21 @@ clj
Use your editor's jack-in functionality when developing with an editor like Emacs, VSCode Calva, or IntelliJ Cursive.
To comple a binary:
### To comple a binary using GraalVM:
```bash
clj -Anative-image
```
To build a Docker image:
Note that you need to install [GraalVM](https://github.com/oracle/graal/releases) and associated libs:
```
sudo apt-get install build-essential libz-dev zlib1g-dev
```
Add `/path/to/graal-vm/bin` to your `$PATH`, and a `$GRAALVM_HOME` to point to `/path/to/graal-vm`.
### To build a Docker image:
Ensure that the `libsunec.so` command is present in the project root.
You can copy it from your GraalVM installation.
......
......@@ -17,7 +17,7 @@
:extra-deps
{clj.native-image
{:git/url "https://github.com/taylorwood/clj.native-image.git"
:sha "567176ddb0f7507c8b0969e0a10f60f848afaf7d"}}}
:sha "7708e7fd4572459c81f6a6b8e44c96f41cdd92d4"}}}
:uberjar {:extra-deps {seancorfield/depstar {:mvn/version "RELEASE"}}
:main-opts ["-m" "hf.depstar.uberjar" "containinator.jar"]}}}
......@@ -9,10 +9,12 @@
(defn http-opts
[{:keys [token form-params]}]
[{:keys [token form-params query-params]}]
(cond-> {:accept :json
:headers {"PRIVATE-TOKEN" token}}
form-params (assoc :form-params form-params)))
:headers {"PRIVATE-TOKEN" token}
:throw-exceptions false}
form-params (assoc :form-params form-params)
query-params (assoc :query-params query-params)))
(defn http-wrap
......@@ -31,61 +33,51 @@
(def http-delete (partial http-wrap http/delete))
(defn process-args
"Takes command line arguments and processes them"
[args]
(update args :gitlab #(str % "/api/v4")))
(defn tags!
"Fetch the tags of a registry repository from GitLab"
(defn fetch!
"Fetch resources given args and link. Lazily concatinate successive pages until
no more pages are available. args should contain :gitlab (api-url) and :token."
[args link]
(let [resp (http-get link args)
tags (:body resp)
resources (:body resp)
next (->> resp :links :next :href)]
(lazy-cat tags
(some->> next (tags! args)))))
(lazy-cat resources
;; Bug in gitlab causes next link to be equal to current page creating an endless loop
(when (not= next link)
(some->> next (fetch! args))))))
(defn fetch-tags!
"Fetch tags lazily"
[args project-id registry-id]
(->> (str (:gitlab args) "/projects/" project-id "/registry/repositories/" registry-id "/tags")
(tags! args)
(lazy-cat [])))
(defn lazy-fetch!
"Fetch resources given args and link. Lazily concatinate successive pages until
no more pages are available. args should contain :gitlab (api-url) and :token.
Only start fetching when sequence is consumed. (take 0 (lazy-fetch ...)) does not
execute any fetches."
[args path]
(lazy-cat []
(fetch! args (str (:gitlab args) path))))
(defn registries!
[args project-id link]
(let [resp (http-get link args)
registries (->> (:body resp)
(map #(select-keys % [:id :path :name]))
(map #(->> (fetch-tags! args project-id (:id %))
(assoc % :tags))))
next (->> resp :links :next :href)]
(lazy-cat registries
(some->> next (registries! args project-id)))))
(defn process-args
"Takes command line arguments and processes them"
[args]
(-> (update args :gitlab #(str % "/api/v4"))
(update :fields #(concat [:id :name] %))))
(defn fetch-registries!
"Lazily fetch registries"
"Lazily fetch registries and recursively their tags"
[args project-id]
(->> (str (:gitlab args) "/projects/" project-id "/registry/repositories")
(registries! args project-id)
(lazy-cat [])))
(let [path (str "/projects/" project-id "/registry/repositories")]
(->> (lazy-fetch! args path)
(map #(->> (str path "/" (:id %) "/tags")
(lazy-fetch! args)
(assoc % :tags))))))
(defn projects!
[args link]
(let [resp (http-get link args)
projects (->> (:body resp)
(filter :container_registry_enabled)
(map #(select-keys % [:id :name :web_url :container_registry_enabled]))
(map #(->> (fetch-registries! args (:id %))
(assoc % :registries))))
next (->> resp :links :next :href)]
(lazy-cat projects
(some->> next (projects! args)))))
(defn fetch-pipelines!
"Lazily fetch jobs"
[args project-id]
(->> (str "/projects/" project-id "/pipelines")
(lazy-fetch! args)))
(defn disable-container-registry!
......@@ -116,9 +108,7 @@
(defn fetch-projects!
"Lazily fetch projects"
[args]
(lazy-cat
[]
(projects! args (str (:gitlab args) "/projects"))))
(lazy-fetch! args "/projects"))
(defn sort-projects
......@@ -127,6 +117,7 @@
(case sort-key
:none projects
:tag-count (sort-by (comp - :tag-count) projects)
:pipeline-count (sort-by (comp - :pipeline-count) projects)
(sort-by sort-key projects)))
......@@ -138,33 +129,48 @@
[args]
(println "Locating projects for given arguments on" (:gitlab args)
"Please be patient, it could take some time." \newline)
(let [args (process-args args)
(let [{:keys [sort limit fields] :as args} (process-args args)
projects (cond->> (fetch-projects! args)
true
(map #(->> (:registries %)
count
(assoc % :registry-count)))
(map #(->> (if (:container_registry_enabled %)
(fetch-registries! args (:id %))
[])
(assoc % :registries)))
true
(map #(assoc % :pipelines (fetch-pipelines! args (:id %))))
(or (some #{:tag-count} fields)
(= sort :tag-count))
(map #(->> (:registries %)
(map (fn [registry]
(count (:tags registry))))
(reduce +)
(assoc % :tag-count)))
(or (some #{:pipeline-count} fields)
(= sort :pipeline-count))
(map #(->> (:pipelines %)
count
(assoc % :pipeline-count)))
true
(sort-projects (:sort args))
(:limit args)
(take (:limit args)))]
limit
(take limit)
true
(map #(assoc % :name (:name_with_namespace %))))]
(pprint/print-table
[:id :name :registry-count :tag-count]
fields
projects)))
(defn terminate!
(defn terminate-tags!
"Instructs GitLab to delete container image tags for all projects and their registries. Supplied
args determines :keep_n number of tags to keep, :older_than age for which to delete, and name_regex
to match tags to delete. In combination these args determines which tags should be matched for deletion."
......@@ -179,6 +185,25 @@
(delete-tags! args (:id project) (:id registry))))))
(defn terminate-pipelines!
[args]
(let [args (process-args args)
projects (->> (fetch-projects! args)
(map #(->> (:id %)
(fetch-pipelines! args)
reverse
(assoc % :pipelines )))
(filter (comp not-empty :pipelines)))]
(println "Initiating pipeline termination.")
(doseq [project projects]
(let [to-drop (->> (:pipelines project)
(drop-last (:keep_n args)))]
(when (not-empty to-drop)
(println (:name_with_namespace project) "-" (-> project :pipelines last :id))
(println "Dropping" (count to-drop) "pipelines from" (-> to-drop first :id) "to" (-> to-drop last :id)))
(doseq [pipeline to-drop])))))
(def app-config
{:app {:command "containinator"
:description "A command to locate and terminate Docker image tags on GitLab servers"
......@@ -201,15 +226,20 @@
:opts [{:option "sort"
:short "s"
:as "Sort output by property"
:type #{:id :name :tag-count}
:type #{:id :name :tag-count :pipeline-count}
:default :none}
{:option "limit"
:short "l"
:as "Limit number of projects to locate"
:type :int}]}
{:command "terminate"
:type :int}
{:option "fields"
:short "f"
:as "The fields to display"
:type #{:tag-count :pipeline-count}
:multiple true}]}
{:command "terminate-tags"
:description "Terminate Docker image tags in GitLab projects matching supplied arguments"
:runs terminate!
:runs terminate-tags!
:opts [{:option "keep_n"
:short "k"
:as "Number of tags to keep"
......@@ -224,7 +254,15 @@
:short "n"
:as "Delete tags matching regex"
:type :string
:default ".*"}]}]})
:default ".*"}]}
{:command "terminate-pipelines"
:description "Terminate GitLab jobs (log and artifacts) matching supplied arguments"
:runs terminate-pipelines!
:opts [{:option "keep_n"
:short "k"
:as "Number of jobs to keep"
:type :int
:default 300}]}]})
(defn -main
......@@ -235,5 +273,12 @@
;; Rich comments
(comment
(concat [:a :b :c] [:j :k :l])
(def token "DC4QLEbKziLhPfjeoWF5")
(locate! {:token token :gitlab "https://gitlab.nsd.no" :limit nil :fields [:pipeline-count] :sort :pipeline-count})
(terminate-pipelines! {:token token :gitlab "https://gitlab.nsd.no" :keep_n 10})
(fetch-pipelines! (process-args {:token token :gitlab "https://gitlab.nsd.no" :query-params {"sort" "asc"
"order_by" "id"}}) 176)
(cli/assert-cfg-sanity app-config)
(cli/parse-cmds ["-t" "foo" "-g" "https://gitlab.example.com" "terminate"] app-config))
(cli/parse-cmds ["-t" "foo" "-g" "https://gitlab.example.com" "terminate"] app-config)
(cli/parse-cmds ["-t" "foo" "-g" "https://gitlab.nsd.no" "locate"] app-config))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment