diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eac469301bb678ab8c2e264791edafcc29f25ca8..22a2e53469d64b6b74490e6b2916f6779772a3e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,18 +1,25 @@
-image: ruby:2.4
+image: ruby:2
+
+# Cache gems in between builds
+cache:
+  key: ${CI_COMMIT_REF_SLUG}
+  paths:
+    - vendor/ruby
 
-variables:
-  PROJECTS: "funkwhale/funkwhale funkwhale/funkwhale.audio"
 
 stages:
   - triage
 
+
+before_script:
+  - ruby -v                                   # Print out ruby version for debugging
+  - bundle install gitlab-triage
+
 dry-run:triage:
   stage: triage
   script:
-    - export PROJECTS_ARRAY=($PROJECTS)
-    - gem install gitlab-triage
-    - gitlab-triage --help
-    - for proj in "${PROJECTS[@]}"; do echo "Triaging $proj" && gitlab-triage --dry-run --token $API_TOKEN --source projects --source-id $proj --host-url https://dev.funkwhale.audio; done
+    - bundle exec gitlab-triage --help
+    - bundle exec gitlab-triage --dry-run --token $API_TOKEN --source group --source-id funkwhale --host-url https://dev.funkwhale.audio
   when: manual
   except:
     - schedules
@@ -20,7 +27,6 @@ dry-run:triage:
 run:triage:
   stage: triage
   script:
-    - gem install gitlab-triage
-    - gitlab-triage --dry-run --token $API_TOKEN --source projects --source-id $CI_PROJECT_PATH --host-url https://dev.funkwhale.audio
+    - bundle exec gitlab-triage --token $API_TOKEN --source group --source-id funkwhale --host-url https://dev.funkwhale.audio
   only:
     - schedules