Improve the workflow for Windows build (#93)

.github/workflows/cmake-darwin.yml

@@ -6,6 +6,7 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
 
 env:
   BUILD_TYPE: Release
@@ -17,14 +18,64 @@ jobs:
     - name: Install Go
       uses: actions/[email protected]
       with:
-        go-version: '>=1.17.0'
+        go-version: '1.22.5'
     - run: go version
     - uses: actions/checkout@v3
     - name: Install OpenMP
-      run: brew install libomp
+      run: |
+        brew install libomp
+        ls $(brew --prefix)/opt/libomp
+    - name: Build ffmpeg
+      run: |
+        cd third_party
+        bash install_ffmpeg.sh
+        rm ffmpeg/version
     - name: Configure CMake
-      run: OpenMP_ROOT=$(brew --prefix)/opt/libomp cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: OpenMP_ROOT=$(brew --prefix)/opt/libomp cmake 
+             -B ${{github.workspace}}/build
+             -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+             -DWITH_FFMPEG=ON
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+    - name: Copy dependencies
+      run: cp $(brew --prefix)/opt/libomp/lib/libomp.dylib ${{github.workspace}}/build/
+    - run: install_name_tool
+             -change "$(brew --prefix)/opt/libomp/lib/libomp.dylib" 
+             '@loader_path/libomp.dylib'
+             ${{github.workspace}}/build/libllm.dylib
+    - uses: actions/upload-artifact@v4
+      with:
+        name: darwin-build
+        path: ${{github.workspace}}/build
     - name: Test
       run: ctest --verbose -C ${{env.BUILD_TYPE}} --test-dir ${{github.workspace}}/build/src/libllm
+
+  release:
+    permissions:
+      contents: write
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: download
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+    - name: Create release package
+      run: ls -lh artifacts/darwin-build/ &&
+           cd artifacts/darwin-build/ &&
+           chmod +x llm &&
+           tar -cvzf release-darwin-arm64.tar.gz
+               llm
+               libllm.dylib
+               libllmpluginffmpeg.dylib
+               libomp.dylib
+    - uses: actions/upload-artifact@v4
+      with:
+        name: darwin-package
+        path: artifacts/darwin-build/release-darwin-arm64.tar.gz
+    - name: create-release
+      uses: softprops/action-gh-release@v2
+      if: startsWith(github.ref, 'refs/tags/')
+      with:
+        files: |
+          artifacts/darwin-build/release-darwin-arm64.tar.gz

.github/workflows/cmake-linux.yml

@@ -19,34 +19,58 @@ jobs:
     - uses: Jimver/[email protected]
       id: cuda-toolkit
       with:
-        cuda: '12.5.0'
+        cuda: '11.8.0'
         linux-local-args: '["--toolkit"]'
     - uses: actions/setup-go@v5
       with:
         go-version: '1.22.5'
     - run: nvcc -V
-    - run: df -h
-    - uses: actions/checkout@v3   
+    - uses: actions/checkout@v3
     - name: Build ffmpeg
       run: cd third_party && bash install_ffmpeg.sh
+    - name: Install cutlass
+      run: cd third_party && bash install_cutlass.sh
     - name: Configure CMake
       run: cmake -B ${{github.workspace}}/build 
                  -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
                  -DWITH_CUDA=ON
-                 -DENABLE_FFMPEG_PLUGIN=ON
+                 -DWITH_FFMPEG=ON
+                 -DWITH_CUTLASS=ON
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
     - name: Test
       run: ctest --verbose -C ${{env.BUILD_TYPE}} --test-dir ${{github.workspace}}/build/src/libllm
+    - uses: actions/upload-artifact@v4
+      with:
+        name: linux-build
+        path: ${{github.workspace}}/build
+
+  release:
+    permissions:
+      contents: write
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: download
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
     - name: Create release package
-      run: cd ${{github.workspace}}/build/ && zip release-linux-x64-cuda.zip 
+      run: ls -lh artifacts/linux-build/ &&
+           cd artifacts/linux-build/ &&
+           chmod +x llm &&
+           tar -cvzf release-linux-x64-cuda.tar.gz
              llm
              libllm.so
              libllmplugincublas.so
              libllmpluginffmpeg.so
-    - name: Release
+    - uses: actions/upload-artifact@v4
+      with:
+        name: linux-package
+        path: artifacts/linux-build/release-linux-x64-cuda.tar.gz
+    - name: create-release
       uses: softprops/action-gh-release@v2
       if: startsWith(github.ref, 'refs/tags/')
       with:
         files: |
-          ${{github.workspace}}/build/release-linux-x64-cuda.zip 
+          artifacts/linux-build/release-linux-x64-cuda.tar.gz

.github/workflows/cmake-windows.yml

@@ -5,23 +5,98 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
 
 env:
   BUILD_TYPE: Release
 
 jobs:
   build:
-    runs-on: windows-latest
+    runs-on: windows-2019
     steps:
+    - uses: Jimver/[email protected]
+      id: cuda-toolkit
+      with:
+        cuda: '11.8.0'
+        method: 'network'
+        sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "thrust", "visual_studio_integration"]'
     - name: Install Go
       uses: actions/[email protected]
       with:
-        go-version: '>=1.17.0'
+        go-version: '1.22.5'
     - run: go version
     - uses: actions/checkout@v3
+    - name: Install wget
+      run: choco install wget --no-progress 
+    - name: Build ffmpeg
+      run: cd third_party && bash install_ffmpeg.sh
+    - name: Install cutlass
+      run: cd third_party && bash install_cutlass.sh
+    - name: Build ffmpeg plugin for Windows
+      run: gcc -shared -o llmpluginffmpeg.dll
+               -Isrc
+               -Ithird_party/ffmpeg
+               -DLIBLLM_EXPORTS
+               -O2
+               src/libllm/read_audio_ffmpeg.cc
+               -Lthird_party/ffmpeg/libavformat
+               -Lthird_party/ffmpeg/libavcodec
+               -Lthird_party/ffmpeg/libavfilter
+               -Lthird_party/ffmpeg/libavutil
+               -Lthird_party/ffmpeg/libswresample
+               -lavformat
+               -lavcodec
+               -lavfilter
+               -lavutil
+               -lswresample
+               -lbcrypt
+               -static
+      shell: bash
     - name: Configure CMake
-      run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+      run: cmake -B ${{github.workspace}}/build
+                 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+                 -DWITH_CUDA=ON
+                 -DWITH_CUTLASS=ON
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+    - name: Copy dependencies
+      run: |
+        copy llmpluginffmpeg.dll ${{github.workspace}}\build
+        copy c:\windows\system32\vcomp140.dll ${{github.workspace}}\build
+        copy ${{github.workspace}}\build\${{env.BUILD_TYPE}}\llm.exe ${{github.workspace}}\build
+        copy ${{github.workspace}}\build\${{env.BUILD_TYPE}}\llm.dll ${{github.workspace}}\build
+        copy ${{github.workspace}}\build\${{env.BUILD_TYPE}}\llmplugincublas.dll ${{github.workspace}}\build
+    - uses: actions/upload-artifact@v4
+      with:
+        name: windows-build
+        path: ${{github.workspace}}/build
     - name: Test
       run: ${{github.workspace}}\build\${{env.BUILD_TYPE}}\unittest.exe
+
+  release:
+    permissions:
+      contents: write  # for softprops/action-gh-release to create a release
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: download
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+    - name: Create release package
+      run: cd artifacts/windows-build/ && zip release-windows-x64-cuda.zip 
+             llm.exe
+             llm.dll
+             llmplugincublas.dll
+             llmpluginffmpeg.dll
+             vcomp140.dll
+    - uses: actions/upload-artifact@v4
+      with:
+        name: windows-package
+        path: artifacts/windows-build/release-windows-x64-cuda.zip
+    - name: create-release
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: softprops/action-gh-release@v2
+      with:
+        files: |
+          artifacts/windows-build/release-windows-x64-cuda.zip

.gitignore

@@ -4,6 +4,8 @@ transformers
 /build
 /run
 /model
+/third_party/ffmpeg
+/third_party/cutlass
 
 __pycache__
 *.model

CMakeLists.txt

@@ -6,7 +6,7 @@ option(WITH_CUDA "CUDA compute support." OFF)
 option(WITH_MKL "MKL support (only for reference)." OFF)
 option(WITH_OPENMP "Build with OpenMP." ON)
 option(WITH_CUTLASS "build MatMul operators with CUTLASS." OFF)
-option(ENABLE_FFMPEG_PLUGIN "build the ffmpeg read audio plugin." OFF)
+option(WITH_FFMPEG "build the ffmpeg read audio plugin." OFF)
 option(MKL_PREFIX "Prefix for MKL headers and libraries." "/opt/intel/mkl")
 
 find_package(OpenMP)
@@ -30,6 +30,15 @@ endif()
 
 if(WITH_CUTLASS)
     add_definitions("-DLIBLLM_CUTLASS_ENABLED")
+    include_directories(${CUTLASS_ROOT}/include)
+
+    set(CUTLASS_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/third_party/cutlass")
+    if (NOT EXISTS ${CUTLASS_ROOT}/include/cutlass/blas3.h)
+        message(FATAL_ERROR 
+          "invalid CUTLASS_ROOT \"${CUTLASS_ROOT}\": unable to find file "
+          "${CUTLASS_ROOT}/include/cutlass/blas3.h, please "
+          "call install_cutlass.sh in third_party first.")
+    endif()
 endif()
 
 if(WITH_MKL)
@@ -44,15 +53,6 @@ if(WITH_MKL)
     endif()
 endif()
 
-if (DEFINED CUTLASS_ROOT)
-    include_directories(${CUTLASS_ROOT}/include)
-    if (NOT EXISTS ${CUTLASS_ROOT}/include/cutlass/blas3.h)
-        message(FATAL_ERROR 
-          "invalid CUTLASS_ROOT \"${CUTLASS_ROOT}\": unable to find file "
-          "${CUTLASS_ROOT}/include/cutlass/blas3.h, please "
-          "enusre CUTLASS_ROOT contains the cutlass \"include\" directory.")
-    endif()
-endif()
 
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@@ -132,16 +132,16 @@ if (WITH_CUDA)
     add_library(llmplugincublas SHARED $<TARGET_OBJECTS:llmplugincublas_static>)
     target_include_directories(llmplugincublas PRIVATE ${libllm_INCDIR})
     target_link_libraries(llmplugincublas lut CUDA::cublas)
-    if(UNIX)
+    if(UNIX AND NOT APPLE)
         target_link_options(llmplugincublas PUBLIC "-Wl,--no-undefined")
-    endif(UNIX)
+    endif()
 endif()
 
-if (ENABLE_FFMPEG_PLUGIN)
+if (WITH_FFMPEG)
     add_library(llmpluginffmpeg SHARED $<TARGET_OBJECTS:llmpluginffmpeg_static>)
     target_include_directories(llmpluginffmpeg PRIVATE ${libllm_INCDIR})
 
-    set(ffmpeg_ROOT "third_party/ffmpeg-7.0.2")
+    set(ffmpeg_ROOT "third_party/ffmpeg")
     target_link_directories(llmpluginffmpeg PUBLIC
         "${ffmpeg_ROOT}/libavformat"
         "${ffmpeg_ROOT}/libavcodec"
@@ -150,9 +150,9 @@ if (ENABLE_FFMPEG_PLUGIN)
         "${ffmpeg_ROOT}/libswresample")
     target_link_libraries(
         llmpluginffmpeg avformat avcodec avfilter avutil swresample ${CMAKE_THREAD_LIBS_INIT}) 
-    if(UNIX)
+    if(UNIX AND NOT APPLE)
         target_link_options(llmpluginffmpeg PUBLIC "-Wl,--no-undefined")
-    endif(UNIX)
+    endif()
 endif()
 
 enable_testing()

go/bin/chat.go

@@ -74,6 +74,8 @@ func chatMain(args []string) {
 	fmt.Println(gLocalizer.Get(MsgInputQuestionSys))
 
 	history := []skill.Message{}
+
+	// TODO: get system prompt for different models
 	systemPrompt := ""
 	for {
 		reader := bufio.NewReader(os.Stdin)