From 8d2ee4c2059e224396510663977fef5dcbd00214 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:00:05 +0100 Subject: [PATCH 1/9] Add refspec #89 --- spaceros/Earthfile | 25 +++++++++++++++++++++---- spaceros/README.md | 6 ++++++ spaceros/build.sh | 1 - 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 442343d..7d3190d 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -14,6 +14,15 @@ VERSION 0.6 FROM ubuntu:jammy +# current git branch, prefilled by earthly +# https://docs.earthly.dev/docs/earthfile/builtin-args +ARG EARTHLY_GIT_BRANCH + +# can be overriden from cli by passing --GIT_BRANCH=... as an argument +# example -> `earthly +image --SPACEROS_REPO_URL=https://... +ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} +ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -21,7 +30,7 @@ earthfile: setup: # Disable prompting during package installation ARG DEBIAN_FRONTEND=noninteractive - + # The following commands are based on the source install for ROS 2 Rolling Ridley. # See: https://docs.ros.org/en/ros2_documentation/rolling/Installation/Ubuntu-Development-Setup.html # The main variation is getting Space ROS sources instead of the Rolling sources. @@ -74,7 +83,6 @@ setup: ENV HOME_DIR=/home/${USERNAME} ENV SPACEROS_DIR=${HOME_DIR}/spaceros ARG IKOS_DIR=${HOME_DIR}/ikos - ARG REPOS_FILE_URL="https://raw.githubusercontent.com/space-ros/space-ros/main/ros2.repos" ENV ROSDISTRO=humble # Create a spaceros user @@ -94,9 +102,18 @@ setup: sources: FROM +setup - COPY github.com/space-ros/space-ros:earthly-wrapper+repos-file/ros2.repos ros2.repos + RUN mkdir spaceros_repo_dir + + IF[ -z "$(git -C spaceros_repo_dir ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] + RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" + GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ + ELSE + RUN echo "cloning branch '${GIT_BRANCH}' from spaceros repo" + GIT CLONE --branch ${GIT_BRANCH} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + END + RUN mkdir src - RUN vcs import src < ros2.repos + RUN vcs import src < spaceros_repo_dir/ros2.repos SAVE ARTIFACT src AS LOCAL src workspace: diff --git a/spaceros/README.md b/spaceros/README.md index 1abf83e..7586866 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -15,6 +15,12 @@ $ ./build.sh The build process will take about 20 or 30 minutes, depending on the host computer. +The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). +Example: +```bash +earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" +``` + ## Running the Space ROS Docker Image in a Container After building the image, you can see the newly-built image by running: diff --git a/spaceros/build.sh b/spaceros/build.sh index fa877b1..5a6b2e8 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -11,7 +11,6 @@ echo "##### Building Space ROS Docker Image #####" echo "" rm -rf src -earthly +sources earthly +image \ --VCS_REF="$VCS_REF" From d7c919e082c9e85586bc7cbbc3440c16fa4a629a Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:17:57 +0100 Subject: [PATCH 2/9] simplify comments --- spaceros/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 7d3190d..33c8a85 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,10 +18,10 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# can be overriden from cli by passing --GIT_BRANCH=... as an argument +# these can be overriden from cli # example -> `earthly +image --SPACEROS_REPO_URL=https://... -ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" +ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} earthfile: COPY Earthfile Earthfile From 664c46e4c4f465ed47429a0782f20a800cecb3b6 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:18:49 +0100 Subject: [PATCH 3/9] simplify comments further --- spaceros/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 33c8a85..102610c 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,8 +18,8 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# these can be overriden from cli -# example -> `earthly +image --SPACEROS_REPO_URL=https://... +# these can be overriden from cli. Example: +# `earthly +image --SPACEROS_REPO_URL=https://...` ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} From cfb2b8ba62d63115a2721ada5fd5ace6e76e1cb3 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Sun, 17 Dec 2023 15:33:43 +0100 Subject: [PATCH 4/9] revert change to build.sh --- spaceros/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/build.sh b/spaceros/build.sh index 5a6b2e8..fa877b1 100755 --- a/spaceros/build.sh +++ b/spaceros/build.sh @@ -11,6 +11,7 @@ echo "##### Building Space ROS Docker Image #####" echo "" rm -rf src +earthly +sources earthly +image \ --VCS_REF="$VCS_REF" From cb59989356e4be6e61c9aefd3bb7e90792811827 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Mon, 18 Dec 2023 08:12:40 +0100 Subject: [PATCH 5/9] simplify git ls-remote command --- spaceros/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 102610c..c2a77ab 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -104,7 +104,7 @@ sources: FROM +setup RUN mkdir spaceros_repo_dir - IF[ -z "$(git -C spaceros_repo_dir ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] + IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ ELSE From 7335a8c67ac59a9c50aeb609487bfa80116fc166 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Mon, 18 Dec 2023 16:59:28 +0100 Subject: [PATCH 6/9] temp changes --- spaceros/Earthfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index c2a77ab..d43003d 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -21,7 +21,7 @@ ARG EARTHLY_GIT_BRANCH # these can be overriden from cli. Example: # `earthly +image --SPACEROS_REPO_URL=https://...` ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" -ARG GIT_BRANCH=${EARTHLY_GIT_BRANCH} +ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} earthfile: COPY Earthfile Earthfile @@ -104,12 +104,12 @@ sources: FROM +setup RUN mkdir spaceros_repo_dir - IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${GIT_BRANCH})" ] - RUN echo "branch '${GIT_BRANCH}' does not exist in spaceros repo, cloning main branch" + IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF})" ] + # branch '${SPACEROS_GIT_REF}' does not exist in spaceros repo, cloning main branch GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ ELSE - RUN echo "cloning branch '${GIT_BRANCH}' from spaceros repo" - GIT CLONE --branch ${GIT_BRANCH} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # branch with the same name exists, cloning from it + GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} spaceros_repo_dir/ END RUN mkdir src From 69ca1bd1961c5647c21fd435b96eff2ac223e1b4 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Tue, 19 Dec 2023 02:53:12 +0100 Subject: [PATCH 7/9] temp --- spaceros/Earthfile | 79 ++++++++++++++++++++++++++++++++++++++-------- spaceros/README.md | 2 +- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index d43003d..dd834df 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -18,11 +18,6 @@ FROM ubuntu:jammy # https://docs.earthly.dev/docs/earthfile/builtin-args ARG EARTHLY_GIT_BRANCH -# these can be overriden from cli. Example: -# `earthly +image --SPACEROS_REPO_URL=https://...` -ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" -ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} - earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -100,20 +95,76 @@ setup: RUN sudo add-apt-repository ppa:kisak/kisak-mesa RUN sudo apt update && sudo apt upgrade -y -sources: - FROM +setup - RUN mkdir spaceros_repo_dir +clone-spaceros-artifacts: + # we must run it in a separate container, so that downstream tasks can be cached if vcs file does not change + FROM ubuntu:jammy + RUN apt-get update && apt-get install -y git wget + + # main purpose of this command is to make sure that the git ls-remote results are not cached + RUN --no-cache echo "Cloning spaceros repo artifacts" + + ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + # set default spaceros git ref to current branch in local repo + ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} # EARTHLY_GIT_BRANCH is populated by earthly + + # if SPACEROS_GIT_REF branch does not exist, artifacts will be cloned from fallback branch + ARG _FALLBACK_BRANCH = "main" + + ARG _GIT_REF_EXISTS = $([ -z $(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF}) ] && echo false || echo true) + ARG _MAIN_SPACEROS_REPO_IS_USED = $([ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] && echo true || echo false) + ARG _GIT_REF_OVERRIDEN_BY_USER = $([ ${SPACEROS_GIT_REF} != ${EARTHLY_GIT_BRANCH} ] && echo true || echo false) + + # If main spaceros repo is used then do it using wget to avoid unneccessary file copies + IF [ ${_MAIN_SPACEROS_REPO_IS_USED} ] && [ ${_GIT_REF_EXISTS} ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${SPACEROS_GIT_REF}/ros2.repos + ELSE IF [ $_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_FALLBACK_BRANCH}/ros2.repos + ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ $_GIT_REF_EXISTS ] + GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} + ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] + GIT CLONE --branch ${_FALLBACK_BRANCH} ${SPACEROS_REPO_URL} + ELSE + RUN exit 1; + END + + - IF[ -z "$(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF})" ] - # branch '${SPACEROS_GIT_REF}' does not exist in spaceros repo, cloning main branch - GIT CLONE ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # ARG _ + + # if branch with the same name does not exist in the spaceros repo, clone from main + # ARG _SPACEROS_GIT_REF_SUBSTITUTED="$( + # if + # then + # echo 'main' + # else + # echo '${SPACEROS_GIT_REF}' + # fi + # ) + # " + + # RUN --no-cache echo $_SPACEROS_GIT_REF_SUBSTITUTED + # $( then echo 'main'; else echo)" + + # we run this command primarily to call `git ls-remote` without caching + + # if branch ${SPACEROS_GIT_REF} does not exist in `spaceros` repo, default to main branch + + # download latest commit. + IF [ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] + + RUN ELSE - # branch with the same name exists, cloning from it - GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} spaceros_repo_dir/ + # otherwise clone whole commit (git clone handles authentication etc) + GIT CLONE --branch ${LATEST_GIT_REF} ${SPACEROS_REPO_URL} . END + SAVE ARTIFACT ros2.repos + +sources: + FROM +setup + COPY +clone-spaceros-artifacts/ros2.repos ros2.repos RUN mkdir src - RUN vcs import src < spaceros_repo_dir/ros2.repos + RUN vcs import src < ros2.repos SAVE ARTIFACT src AS LOCAL src workspace: diff --git a/spaceros/README.md b/spaceros/README.md index 7586866..4cae5ab 100644 --- a/spaceros/README.md +++ b/spaceros/README.md @@ -18,7 +18,7 @@ The build process will take about 20 or 30 minutes, depending on the host comput The build process defaults to cloning the `ros2.repos` file from [spaceros](https://github.com/space-ros/space-ros) repository. It looks for a branch with the same name as the current local branch; if it doesn't find one, it falls back to cloning from the main branch. For testing purposes, you can customize both the spaceros repository URL and the branch name by modifying arguments defined in the [Earthfile](./Earthfile). Example: ```bash -earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" +earthly +image --SPACEROS_REPO_URL="https://github.com/my-org/my-spaceros-fork.git" --SPACEROS_GIT_REF="my-branch-name" ``` ## Running the Space ROS Docker Image in a Container From 6d7cf7493819eaeba2d195f4f69b8c3e42b69768 Mon Sep 17 00:00:00 2001 From: xfiderek Date: Wed, 20 Dec 2023 20:18:15 +0100 Subject: [PATCH 8/9] push working version --- spaceros/Earthfile | 73 +++++++++++----------------------------------- 1 file changed, 17 insertions(+), 56 deletions(-) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index dd834df..6f55344 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -14,10 +14,6 @@ VERSION 0.6 FROM ubuntu:jammy -# current git branch, prefilled by earthly -# https://docs.earthly.dev/docs/earthfile/builtin-args -ARG EARTHLY_GIT_BRANCH - earthfile: COPY Earthfile Earthfile SAVE ARTIFACT Earthfile @@ -95,73 +91,38 @@ setup: RUN sudo add-apt-repository ppa:kisak/kisak-mesa RUN sudo apt update && sudo apt upgrade -y -clone-spaceros-artifacts: +spaceros-artifacts: # we must run it in a separate container, so that downstream tasks can be cached if vcs file does not change FROM ubuntu:jammy RUN apt-get update && apt-get install -y git wget # main purpose of this command is to make sure that the git ls-remote results are not cached RUN --no-cache echo "Cloning spaceros repo artifacts" - - ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" - # set default spaceros git ref to current branch in local repo - ARG SPACEROS_GIT_REF=${EARTHLY_GIT_BRANCH} # EARTHLY_GIT_BRANCH is populated by earthly - - # if SPACEROS_GIT_REF branch does not exist, artifacts will be cloned from fallback branch - ARG _FALLBACK_BRANCH = "main" - - ARG _GIT_REF_EXISTS = $([ -z $(git ls-remote ${SPACEROS_REPO_URL} ${SPACEROS_GIT_REF}) ] && echo false || echo true) - ARG _MAIN_SPACEROS_REPO_IS_USED = $([ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] && echo true || echo false) - ARG _GIT_REF_OVERRIDEN_BY_USER = $([ ${SPACEROS_GIT_REF} != ${EARTHLY_GIT_BRANCH} ] && echo true || echo false) - - # If main spaceros repo is used then do it using wget to avoid unneccessary file copies - IF [ ${_MAIN_SPACEROS_REPO_IS_USED} ] && [ ${_GIT_REF_EXISTS} ] - RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${SPACEROS_GIT_REF}/ros2.repos - ELSE IF [ $_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] - RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_FALLBACK_BRANCH}/ros2.repos - ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ $_GIT_REF_EXISTS ] - GIT CLONE --branch ${SPACEROS_GIT_REF} ${SPACEROS_REPO_URL} - ELSE IF [ !$_MAIN_SPACEROS_REPO_IS_USED ] && [ !$_GIT_REF_EXISTS ] && [ !$_GIT_REF_OVERRIDEN_BY_USER ] - GIT CLONE --branch ${_FALLBACK_BRANCH} ${SPACEROS_REPO_URL} - ELSE - RUN exit 1; - END - - - # ARG _ + # current git branch, prefilled by earthly: https://docs.earthly.dev/docs/earthfile/builtin-args + ARG EARTHLY_GIT_BRANCH - # if branch with the same name does not exist in the spaceros repo, clone from main - # ARG _SPACEROS_GIT_REF_SUBSTITUTED="$( - # if - # then - # echo 'main' - # else - # echo '${SPACEROS_GIT_REF}' - # fi - # ) - # " - - # RUN --no-cache echo $_SPACEROS_GIT_REF_SUBSTITUTED - # $( then echo 'main'; else echo)" - - # we run this command primarily to call `git ls-remote` without caching - - # if branch ${SPACEROS_GIT_REF} does not exist in `spaceros` repo, default to main branch + ARG SPACEROS_REPO_URL="https://github.com/space-ros/space-ros.git" + # if current local branch does not exist in target repo then use main. note that branch supplied from CLI overrides this behavior. + ARG SPACEROS_GIT_REF="$( [ -n \"$(git ls-remote $SPACEROS_REPO_URL $EARTHLY_GIT_BRANCH)\" ] && echo $EARTHLY_GIT_BRANCH || echo 'main' )" + # get exact commit hash. this makes sure that build will be re-triggered when new commit is pushed + ARG _GIT_COMMIT_HASH = "$(git ls-remote $SPACEROS_REPO_URL $SPACEROS_GIT_REF | cut -f 1)" + + # this means that branch specified by user from CLI does not exist + IF [ -z ${_GIT_COMMIT_HASH} ] + RUN echo "Specified branch ${SPACEROS_GIT_REF} does not exist" && exit 1 + END - # download latest commit. - IF [ ${SPACEROS_REPO_URL} = "https://github.com/space-ros/space-ros.git" ] - - RUN + IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] + RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos ELSE - # otherwise clone whole commit (git clone handles authentication etc) - GIT CLONE --branch ${LATEST_GIT_REF} ${SPACEROS_REPO_URL} . + GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} . END SAVE ARTIFACT ros2.repos sources: FROM +setup - COPY +clone-spaceros-artifacts/ros2.repos ros2.repos + COPY +spaceros-artifacts/ros2.repos ros2.repos RUN mkdir src RUN vcs import src < ros2.repos From 3790bdc40582d301a20b5eb5292f94060db3717c Mon Sep 17 00:00:00 2001 From: xfiderek Date: Wed, 20 Dec 2023 20:20:44 +0100 Subject: [PATCH 9/9] add clarification to wget --- spaceros/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/spaceros/Earthfile b/spaceros/Earthfile index 6f55344..192078d 100644 --- a/spaceros/Earthfile +++ b/spaceros/Earthfile @@ -114,6 +114,7 @@ spaceros-artifacts: END IF [ $SPACEROS_REPO_URL = "https://github.com/space-ros/space-ros.git" ] + # run wget because main repo's host is known + it is public RUN wget https://raw.githubusercontent.com/space-ros/space-ros/${_GIT_COMMIT_HASH}/ros2.repos ELSE GIT CLONE --branch ${_GIT_COMMIT_HASH} ${SPACEROS_REPO_URL} .