Skip to content

new-tag-created

new-tag-created #352

Workflow file for this run

permissions:
contents: write
actions: write
issues: write
name: Release
on:
create:
tags:
- 'v*'
- 'commit-*'
repository_dispatch:
types: [new-tag-created]
jobs:
release:
strategy:
matrix:
os: [debian-12, ubuntu-24.04, ubuntu-24.10, Rhino]
include:
- os: debian-12
container: debian:bookworm
- os: ubuntu-24.04
container: ubuntu:noble
- os: ubuntu-24.10
container: ubuntu:oracular
- os: Rhino
container: ghcr.io/rhino-linux/docker:latest
# DEFAULT_USER is rhino at Rhino, root at debian and ubuntu.
# NON_ROOT is builduser
runs-on: ubuntu-latest
container:
image: ${{ matrix.container }}
env:
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
# SCCACHE_RECACHE: 1 # Uncomment this to clear cache, then comment
name: Release-${{ matrix.os }}
steps:
- name: Debug event
run: |
echo "Event name: ${{ github.event_name }}"
echo "Event payload: ${{ toJson(github.event.client_payload) }}"
ls -la $GITHUB_WORKSPACE
- name: install apt-utils and wget and git (Linux)
id: startup
shell: bash
run: |
echo "container_name=${{ matrix.os }}" >> $GITHUB_OUTPUT
if [[ "${{ matrix.os }}" == "debian-12" ]]; then
apt-get update
apt-get install -y sudo wget git
echo "non_root=builduser" >> $GITHUB_OUTPUT
elif [[ "${{ matrix.os }}" == "Rhino" ]]; then
sudo apt-get update
sudo apt-get install -y apt-utils wget software-properties-common
#sudo add-apt-repository ppa:apt-fast/stable
sudo apt-get update
sudo apt-get -y install git
# rhino
USER=`whoami`
echo "non_root=builduser" >> $GITHUB_OUTPUT
else
apt-get update
apt-get install -y sudo apt-utils wget software-properties-common
#sudo add-apt-repository ppa:apt-fast/stable
sudo apt-get update
sudo apt-get -y install git
echo "non_root=builduser" >> $GITHUB_OUTPUT
fi
- uses: actions/checkout@v4
with:
fetch-depth: 0
set-safe-directory: 'true'
- name: debian-12 and ubuntu-23.04 patch
shell: bash
run: |
if [[ "${{ matrix.os }}" == "debian-12" || "${{ matrix.os }}" == "ubuntu-23.04" ]]; then
apt-get install patch
patch -p1 -i debian-12.patch
fi
- name: check
run: |
export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
sudo git config --global --add safe.directory $GITHUB_WORKSPACE
- name: Setup non-root user
shell: bash
run: |
# DEFAULT_USER is rhino at Rhino, root at debian and ubuntu.
# NON_ROOT is builduser
NON_ROOT="${{ steps.startup.outputs.non_root }}"
sudo mkdir -p /github/workspace/
sudo useradd -m -d /github/workspace/ -s /usr/bin/bash ${NON_ROOT}
sudo mkdir -p /github/workspace/.cache
sudo mkdir -p /github/workspace/.rustup
sudo mkdir -p /github/workspace/.cargo
#sudo cp .bazeliskrc /github/workspace/
sudo chown ${NON_ROOT}:${NON_ROOT} /github/workspace/
sudo -u $NON_ROOT touch /github/workspace/.bashrc
sudo chown ${NON_ROOT}:${NON_ROOT} /github/workspace/.bashrc
sudo cp config.toml /github/workspace/.cargo/
sudo chown -R ${NON_ROOT}:${NON_ROOT} /github/workspace/.rustup
sudo chown -R ${NON_ROOT}:${NON_ROOT} /github/workspace/.cargo
DEFAULT_USER=$(whoami)
# bazel cache
export XDG_CACHE_HOME=/github/workspace/.cache/
sudo mkdir -p ${XDG_CACHE_HOME}/bazel
echo "XDG_CACHE_HOME=${XDG_CACHE_HOME}" >> $GITHUB_ENV
echo 'export XDG_CACHE_HOME=${XDG_CACHE_HOME}' | sudo -u ${NON_ROOT} tee -a /github/workspace/.bashrc
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${XDG_CACHE_HOME}
sudo mkdir -p $GITHUB_WORKSPACE/.git
sudo chown -R ${NON_ROOT}:${NON_ROOT} $GITHUB_WORKSPACE/.git
sudo -u ${NON_ROOT} whoami
sudo -u ${NON_ROOT} id
whoami
id
- name: Setup Mozc source
run: |
sudo git clone --filter=tree:0 -b master https://github.com/phoepsilonix/mozc.git source
cd source
sudo git submodule update --init
- name: Setup Variables
id: variables
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
sudo git config --global push.gpgSign if-asked
#cd source
#SHA512SUMS=$(find src -type f -print0 | sort -z | xargs -0 sha512sum | sha512sum | cut -d' ' -f1)
#echo "mozc_checksum=$SHA512SUMS" >> $GITHUB_OUTPUT
#cd ..
TAG=${GITHUB_REF##*/}
if [[ $TAG == "main" ]];then
echo "Running release for branch: ${{ github.event.client_payload.repo.branch }}"
echo "Tag: ${{ github.event.client_payload.repo.tag }}"
TAG="${{ github.event.client_payload.repo.tag }}"
fi
if [[ "$TAG" =~ ^commit- ]]; then
cd source
COMMIT=$(sudo git log -1 ${TAG#commit-} --pretty=%h)
sudo git checkout $COMMIT
echo $TAG
cd ..
#sudo git tag $TAG -d
#sudo git push origin $TAG -d
cd source
tag=$(sudo git describe --tags --always ${COMMIT})
base_tag="${tag%%-*}"
if [[ "$tag" =~ "-" ]];then
after_tag="${tag#*-}"
echo $after_tag
TAG=${base_tag}"-${{ matrix.os }}-"${after_tag}
TAG2=${base_tag}"-${{ matrix.os }}-with-jp-dict-"${after_tag}
else
after_tag=""
TAG=${base_tag}"-${{ matrix.os }}"
TAG2=${base_tag}"-${{ matrix.os }}-with-jp-dict-"${after_tag}
fi
echo $TAG
echo $TAG2
echo $base_tag
echo $after_tag
TAG=${base_tag}"-${{ matrix.os }}-"${after_tag}
TAG2=${base_tag}"-${{ matrix.os }}-with-jp-dict-"${after_tag}
TAG=${TAG%*-}
TAG2=${TAG2%*-}
echo $TAG
echo $TAG2
cd ..
sudo git tag $TAG -d||true
sudo git tag $TAG2 -d||true
sudo git push origin $TAG -d ||true
sudo git push origin $TAG2 -d ||true
sudo git tag $TAG
sudo git tag $TAG2
sudo git push origin $TAG
sudo git push origin $TAG2
echo "tag_name=" $TAG
echo "tag_name_2=" $TAG2
echo "tag_name=${TAG}" >> $GITHUB_OUTPUT
echo "tag_name_2=${TAG2}" >> $GITHUB_OUTPUT
echo "tag_commit=$(sudo git log -1 --pretty=%H ${TAG})" >> $GITHUB_OUTPUT
echo "base_tag=${base_tag}" >> $GITHUB_OUTPUT
else
echo $TAG
tag=$TAG
base_tag="${tag%%-*}"
after_tag="${tag#*-${{ matrix.os }}}"
TAG=${base_tag}"-${{ matrix.os }}-"${after_tag}
TAG2=${base_tag}"-${{ matrix.os }}-with-jp-dict-"${after_tag}
TAG=${TAG%*-}
TAG2=${TAG2%*-}
sudo git tag $TAG -d||true
sudo git tag $TAG2 -d||true
sudo git push origin $TAG -d ||true
sudo git push origin $TAG2 -d ||true
sudo git tag $TAG
sudo git tag $TAG2
sudo git push origin $TAG
sudo git push origin $TAG2
cd source
COMMIT=$(sudo git log -1 --pretty=%H $base_tag$after_tag)
echo $COMMIT
sudo git checkout $COMMIT
echo "tag_name=${TAG}" >> $GITHUB_OUTPUT
echo "tag_name_2=${TAG2}" >> $GITHUB_OUTPUT
echo "tag_commit=$(sudo git log -1 --pretty=%H ${COMMIT})" >> $GITHUB_OUTPUT
echo "base_tag=${base_tag}" >> $GITHUB_OUTPUT
fi
- name: check variables
run: |
echo "${{ steps.variables.outputs.tag_name }}"
echo "${{ steps.variables.outputs.tag_name_2 }}"
echo "${{ steps.variables.outputs.tag_commit }}"
echo "${{ steps.variables.outputs.base_tag }}"
#echo "${{ steps.variables.outputs.mozc_checksum }}"
- name: install build tools (Linux)
shell: bash
run: |
if [[ "${{ matrix.os }}" == "debian-12" ]]; then
sudo sed 's/Types: deb *$/Types: deb deb-src/' -i /etc/apt/sources.list.d/debian.sources
elif [[ "${{ matrix.os }}" == "ubuntu-24.04" || "${{ matrix.os }}" == "Rhino" ]]; then
sudo sed 's/Types: deb *$/Types: deb deb-src/' -i /etc/apt/sources.list.d/ubuntu.sources
else
sudo sed 's/Types: deb *$/Types: deb deb-src/' -i /etc/apt/sources.list.d/ubuntu.sources || true
sudo sed 's/^.*deb-src /deb-src /' -i /etc/apt/sources.list || true
fi
sudo apt-get update
sudo apt-get build-dep -y mozc
sudo apt-get install -y build-essential dpkg-dev make qt6ct qt6-base-dev libfcitx5-qt6-dev curl unzip fakeroot
#if [[ "${{ matrix.os }}" == "Rhino" ]]; then
sudo apt-get install -y python3-six
#fi
- name: Install mold linker
uses: rui314/setup-mold@staging
with:
mold-version: 2.36.0
make-default: true
- name: Install bazel
env:
USE_BAZEL_VERSION: "8.0.1"
run: |
# cache cleanup
sudo rm -rf /github/workspace/.cache/bazel/*
NON_ROOT="${{ steps.startup.outputs.non_root }}"
sudo mkdir -p /usr/local/bin
sudo curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64
#sudo curl -Lo /usr/local/bin/bazel https://github.com/bazelbuild/bazel/releases/download/8.0.1/bazel-8.0.1-linux-x86_64
sudo chmod +x /usr/local/bin/bazel
sudo mkdir -p /github/workspace/.cache/bazelisk
sudo chown -R ${NON_ROOT}:${NON_ROOT} /github/workspace/.cache
sudo -u ${NON_ROOT} bazel --version
DEFAULT_USER=$(whoami)
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} /github/workspace/.cache/bazel/
- name: Bazel Cache
uses: actions/cache@v4
with:
path: |
${{ env.XDG_CACHE_HOME }}bazel
key: bazel-${{ steps.startup.outputs.container_name }}-${{ steps.variables.outputs.base_tag }}-${{ steps.variables.outputs.tag_commit }}
restore-keys: |
bazel-${{ steps.startup.outputs.container_name }}-${{ steps.variables.outputs.base_tag }}-
bazel-${{ steps.startup.outputs.container_name }}-
- name: Build(normal)
env:
USE_BAZEL_VERSION: "8.0.1"
shell: bash
run: |
NON_ROOT="${{ steps.startup.outputs.non_root }}"
export PATH=$HOME/bin:$PATH
sudo cp -a debian source/
sudo chown -R ${NON_ROOT}:${NON_ROOT} .
sudo chown -R ${NON_ROOT}:${NON_ROOT} /github/workspace/.cache
cd source
echo $PWD
ls -la
sudo sed 's/bazel clean/bazel --version/' debian/rules -i
sudo sed "s/^\(LDFLAGS:=.*\)/\1 -fuse-ld=mold/" debian/rules -i
sudo sed -e '/register_toolchains("@androidndk\/\/:all")/d' -i src/MODULE.bazel
. /github/workspace/.bashrc
sudo chown -R ${NON_ROOT}:${NON_ROOT} .
sudo -u ${NON_ROOT} dpkg-buildpackage -rfakeroot -b --no-sign
cd ..
sudo -u ${NON_ROOT} mkdir -p release
sudo -u ${NON_ROOT} mv *deb mozc_* release/
- name: Generate release note
id: release_note
shell: bash
run: |
# DEFAULT_USER is Rhino(rhino) or root(debian and ubuntu)
DEFAULT_USER=$(whoami)
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} .
ls -la
git checkout main
git checkout HEAD~ -- last_commit.txt
OLD_COMMIT=$(cat last_commit.txt)
git checkout HEAD -- last_commit.txt
NEW_COMMIT=$(cat last_commit.txt)
cd source
if [[ "$OLD_COMMIT" == "$NEW_COMMIT" ]]; then
NOTE=$(git log -1)
else
#NOTE=$(git log $OLD_COMMIT~..$NEW_COMMIT)
NOTE=$(git log -1)
fi
{
echo "release_note<<EOF"
echo "${NOTE}"
echo "EOF"
} >> $GITHUB_OUTPUT
- name: Create release
uses: ncipollo/release-action@v1
with:
tag: "${{ steps.variables.outputs.tag_name }}"
name: "${{ steps.variables.outputs.tag_name }}"
artifacts: 'release/*'
token: ${{ secrets.GITHUB_TOKEN }}
body: "${{ steps.release_note.outputs.release_note }}"
- name: Added SudachiDict for -with-jp-dict
run: |
# DEFAULT_USER is Rhino(rhino) or root(debian and ubuntu)
DEFAULT_USER=$(whoami)
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} .
srcdir=$PWD/source/
curl -LO https://github.com/phoepsilonix/dict-to-mozc/releases/download/v0.6.14/dict-to-mozc-x86_64-unknown-linux-gnu.tar.xz
tar xvf dict-to-mozc-x86_64-unknown-linux-gnu.tar.xz --strip-components=1
sudo cp dict-to-mozc /usr/local/bin/
_sudachidict_date=$(curl -s 'http://sudachi.s3-website-ap-northeast-1.amazonaws.com/sudachidict-raw/' | grep -o '<td>[0-9]*</td>' | grep -o '[0-9]*' | sort -n | tail -n 1)
echo $_sudachidict_date
curl -LO "http://sudachi.s3-website-ap-northeast-1.amazonaws.com/sudachidict-raw/${_sudachidict_date}/small_lex.zip"
curl -LO "http://sudachi.s3-website-ap-northeast-1.amazonaws.com/sudachidict-raw/${_sudachidict_date}/core_lex.zip"
curl -LO "http://sudachi.s3-website-ap-northeast-1.amazonaws.com/sudachidict-raw/${_sudachidict_date}/notcore_lex.zip"
unzip -o small_lex.zip
unzip -o core_lex.zip
unzip -o notcore_lex.zip
# すだちを優先
echo '1. Build the rust program(dict-to-mozc), it may take some time...'
which dict-to-mozc
echo '2. Convert SudachiDict to Mozc System Dictionary format. It may take some time...'
cat "${srcdir}"/src/data/dictionary_oss/dictionary*.txt > all-dict.txt
cat small_lex.csv core_lex.csv notcore_lex.csv > all.csv
cp ${srcdir}/src/data/dictionary_oss/id.def ./
dict-to-mozc -s -i ./id.def -f all.csv >> all-dict.txt
echo '3. Convert Mecab-unidic-neologd to Mozc System Dictionary format. It may take some time...'
curl -LO https://github.com/phoepsilonix/mecab-unidic-neologd/raw/refs/heads/master/seed/mecab-unidic-user-dict-seed.20200910.csv.xz
xz -d mecab-unidic-user-dict-seed.20200910.csv.xz
dict-to-mozc -n -i ./id.def -f mecab-unidic-user-dict-seed.20200910.csv >> all-dict.txt
echo '4. Convert Mecab-ipadic-neologd to Mozc System Dictionary format. It may take some time...'
curl -LO https://github.com/phoepsilonix/mecab-ipadic-neologd/raw/refs/heads/master/seed/mecab-user-dict-seed.20200910.csv.xz
xz -d mecab-user-dict-seed.20200910.csv.xz
dict-to-mozc -n -P 12 -N 10 -i ./id.def -f mecab-user-dict-seed.20200910.csv >> all-dict.txt
echo '5. Duplicate data will be removed.'
# 読み、品詞ID、右品詞ID、表記による重複チェック
awk 'BEGIN{FS="\t";OFS="\t"}{if (!a[$1,$2,$3,$5]++) {print $0}}' all-dict.txt|sort > finish-dict.txt
echo '6. Finally, add the SudachiDict dictionary to the Mozc source.'
mkdir -p tmp
cd tmp
ROWS=$(echo $(($(wc -l ../finish-dict.txt |cut -f1 -d" ")/10+1)))
split -d -l $ROWS --additional-suffix=.txt ../finish-dict.txt dictionary
cp -af ./dictionary*.txt "$srcdir/src/data/dictionary_oss/"
sync
- name: Build(-with-jp-dict)
env:
USE_BAZEL_VERSION: "8.0.1"
shell: bash
run: |
NON_ROOT="${{ steps.startup.outputs.non_root }}"
sudo chown -R ${NON_ROOT}:${NON_ROOT} ${{ env.XDG_CACHE_HOME }}
sudo chown -R ${NON_ROOT}:${NON_ROOT} .
rm -rf *deb mozc_*
export PATH=$HOME/bin:$PATH
#cp -a debian source/
cd source
sudo -u ${NON_ROOT} patch -p1 -i ../dict-license.patch
#sed 's/bazel clean/bazel --version/' debian/rules -i
#sed "s/^\(LDFLAGS:=.*\)/\1 -fuse-ld=mold/" debian/rules -i
. /github/workspace/.bashrc
sudo chown -R ${NON_ROOT}:${NON_ROOT} .
sudo -u ${NON_ROOT} dpkg-buildpackage -rfakeroot -b --no-sign
cd ..
sudo -u ${NON_ROOT} mkdir -p release_2
sudo -u ${NON_ROOT} mv *deb mozc_* release_2/
# DEFAULT_USER is Rhino(rhino) or root(debian and ubuntu)
if [[ "${{ matrix.os }}" == "Rhino" ]]; then
DEFAULT_USER=$(whoami)
echo $DEFAULT_USER
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${{ env.XDG_CACHE_HOME }}
else
DEFAULT_USER=$(whoami)
echo $DEFAULT_USER
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} ${{ env.XDG_CACHE_HOME }}
fi
- name: Generate release note
id: release_note_2
shell: bash
run: |
# DEFAULT_USER is Rhino(rhino) or root(debian and ubuntu)
DEFAULT_USER=$(whoami)
sudo chown -R ${DEFAULT_USER}:${DEFAULT_USER} .
git checkout main
git checkout HEAD~ -- last_commit.txt
OLD_COMMIT=$(cat last_commit.txt)
git checkout HEAD -- last_commit.txt
NEW_COMMIT=$(cat last_commit.txt)
cd source
if [[ "$OLD_COMMIT" == "$NEW_COMMIT" ]]; then
NOTE=$(git log -1)
else
#NOTE=$(git log $OLD_COMMIT~..$NEW_COMMIT)
NOTE=$(git log -1)
fi
{
echo "release_note<<EOF"
echo "Add Japanese dictionaries for improved tokenization"
echo
echo "- [SudachiDict: Lexicon for Sudachi tokenizer](https://github.com/WorksApplications/SudachiDict)"
echo "- [Neologism dictionary for UniDic-MeCab](https://github.com/neologd/mecab-unidic-neologd/)"
echo "- [Neologism dictionary for IPAdic-MeCab](https://github.com/neologd/mecab-ipadic-neologd/)"
echo
echo "${NOTE}"
echo "EOF"
} >> $GITHUB_OUTPUT
- name: Create release
uses: ncipollo/release-action@v1
with:
tag: "${{ steps.variables.outputs.tag_name_2 }}"
name: "${{ steps.variables.outputs.tag_name_2 }}"
artifacts: 'release_2/*'
token: ${{ secrets.GITHUB_TOKEN }}
body: "${{ steps.release_note_2.outputs.release_note }}"
- name: Delete tag if job failed
if: failure()
run: |
TAG=${GITHUB_REF##*/}
if [[ $TAG == "main" ]];then
echo "Running release for branch: ${{ github.event.client_payload.repo.branch }}"
echo "Tag: ${{ github.event.client_payload.repo.tag }}"
TAG="${{ github.event.client_payload.repo.tag }}"
fi
git push origin -d $TAG || true
gh release delete --cleanup-tag "${{ steps.variables.outputs.tag_name }}" || true
gh release delete --cleanup-tag "${{ steps.variables.outputs.tag_name_2 }}" || true
git push origin -d "${{ steps.variables.outputs.tag_name }}" || true
git push origin -d "${{ steps.variables.outputs.tag_name_2 }}" || true
tag-clear:
runs-on: ubuntu-latest
needs: release
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Configure Git
run: |
git config user.name "Masato TOYOSHIMA"
git config user.email [email protected]
- name: tag deletion
shell: bash
run: |
TAG=${GITHUB_REF##*/}
if [[ "$TAG" == "main" ]];then
TAG="${{ github.event.client_payload.repo.tag }}"
fi
git push origin $TAG -d || true
- name: gh release delete
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
git fetch --tags
OLD_RELEASES=$(git tag --points-at HEAD^)
[[ -z $OLD_RELEASES ]] && OLD_RELEASES=$(git tag --points-at HEAD^^)
[[ -z $OLD_RELEASES ]] && OLD_RELEASES=$(git tag --points-at HEAD^^^)
[[ -z $OLD_RELEASES ]] && OLD_RELEASES=$(git tag --points-at HEAD^^^^)
[[ -z $OLD_RELEASES ]] && OLD_RELEASES=$(git tag --points-at HEAD^^^^^)
[[ -z $OLD_RELEASES ]] && OLD_RELEASES=$(git tag --points-at HEAD^^^^^^)
if [[ "$(echo ${OLD_RELEASES}|grep -Eo "\-([^-]*)-g[^-]*$")" != "" ]];then
echo ${OLD_RELEASES}
echo ${OLD_RELEASES} | grep -E "\-([^-]*)-g[^-]*$" | xargs -n1 gh release delete --cleanup-tag -y || true
fi