llama : handle unknown utf8 bytes

[ggerganov]

fix #6112

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 525 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=8915.27ms p(95)=22153.42ms fails=, finish reason: stop=465 truncated=60
• Prompt processing (pp): avg=110.46tk/s p(95)=548.77tk/s
• Token generation (tg): avg=31.84tk/s p(95)=46.41tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=gg/fix-utf8-to-byte commit=fd8eda19df9aee3652acf3fee73d42b258b11bcf

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 525 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1716897664 --> 1716898290
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 361.59, 361.59, 361.59, 361.59, 361.59, 551.08, 551.08, 551.08, 551.08, 551.08, 544.88, 544.88, 544.88, 544.88, 544.88, 579.35, 579.35, 579.35, 579.35, 579.35, 656.1, 656.1, 656.1, 656.1, 656.1, 676.1, 676.1, 676.1, 676.1, 676.1, 688.06, 688.06, 688.06, 688.06, 688.06, 708.66, 708.66, 708.66, 708.66, 708.66, 738.65, 738.65, 738.65, 738.65, 738.65, 741.18, 741.18, 741.18, 741.18, 741.18, 743.43, 743.43, 743.43, 743.43, 743.43, 762.67, 762.67, 762.67, 762.67, 762.67, 778.63, 778.63, 778.63, 778.63, 778.63, 750.0, 750.0, 750.0, 750.0, 750.0, 768.57, 768.57, 768.57, 768.57, 768.57, 775.31, 775.31, 775.31, 775.31, 775.31, 778.2, 778.2, 778.2, 778.2, 778.2, 781.54, 781.54, 781.54, 781.54, 781.54, 801.5, 801.5, 801.5, 801.5, 801.5, 798.23, 798.23, 798.23, 798.23, 798.23, 797.43, 797.43, 797.43, 797.43, 797.43, 803.01, 803.01, 803.01, 803.01, 803.01, 802.28, 802.28, 802.28, 802.28, 802.28, 807.36, 807.36, 807.36, 807.36, 807.36, 825.01, 825.01, 825.01, 825.01, 825.01, 823.89, 823.89, 823.89, 823.89, 823.89, 828.88, 828.88, 828.88, 828.88, 828.88, 825.94, 825.94, 825.94, 825.94, 825.94, 823.64, 823.64, 823.64, 823.64, 823.64, 823.1, 823.1, 823.1, 823.1, 823.1, 823.64, 823.64, 823.64, 823.64, 823.64, 829.46, 829.46, 829.46, 829.46, 829.46, 826.45, 826.45, 826.45, 826.45, 826.45, 827.4, 827.4, 827.4, 827.4, 827.4, 826.57, 826.57, 826.57, 826.57, 826.57, 838.14, 838.14, 838.14, 838.14, 838.14, 841.16, 841.16, 841.16, 841.16, 841.16, 846.53, 846.53, 846.53, 846.53, 846.53, 843.0, 843.0, 843.0, 843.0, 843.0, 842.08, 842.08, 842.08, 842.08, 842.08, 842.75, 842.75, 842.75, 842.75, 842.75, 845.51, 845.51, 845.51, 845.51, 845.51, 845.05, 845.05, 845.05, 845.05, 845.05, 838.54, 838.54, 838.54, 838.54, 838.54, 792.79, 792.79, 792.79, 792.79, 792.79, 793.23, 793.23, 793.23, 793.23, 793.23, 792.24, 792.24, 792.24, 792.24, 792.24, 794.92, 794.92, 794.92, 794.92, 794.92, 798.07, 798.07, 798.07, 798.07, 798.07, 797.93, 797.93, 797.93, 797.93, 797.93, 800.29, 800.29, 800.29, 800.29, 800.29, 802.54, 802.54, 802.54, 802.54, 802.54, 806.27, 806.27, 806.27, 806.27, 806.27, 808.37, 808.37, 808.37, 808.37, 808.37, 809.25, 809.25, 809.25, 809.25, 809.25, 809.35, 809.35, 809.35, 809.35, 809.35, 809.39, 809.39, 809.39, 809.39, 809.39, 806.55, 806.55, 806.55, 806.55, 806.55, 806.51, 806.51, 806.51, 806.51, 806.51, 808.0, 808.0, 808.0, 808.0, 808.0, 808.33, 808.33, 808.33, 808.33]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 525 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1716897664 --> 1716898290
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 39.92, 39.92, 39.92, 39.92, 39.92, 40.88, 40.88, 40.88, 40.88, 40.88, 28.76, 28.76, 28.76, 28.76, 28.76, 32.65, 32.65, 32.65, 32.65, 32.65, 33.27, 33.27, 33.27, 33.27, 33.27, 34.36, 34.36, 34.36, 34.36, 34.36, 35.34, 35.34, 35.34, 35.34, 35.34, 35.61, 35.61, 35.61, 35.61, 35.61, 35.81, 35.81, 35.81, 35.81, 35.81, 35.53, 35.53, 35.53, 35.53, 35.53, 35.08, 35.08, 35.08, 35.08, 35.08, 34.84, 34.84, 34.84, 34.84, 34.84, 34.18, 34.18, 34.18, 34.18, 34.18, 33.89, 33.89, 33.89, 33.89, 33.89, 33.59, 33.59, 33.59, 33.59, 33.59, 31.91, 31.91, 31.91, 31.91, 31.91, 30.54, 30.54, 30.54, 30.54, 30.54, 30.5, 30.5, 30.5, 30.5, 30.5, 30.81, 30.81, 30.81, 30.81, 30.81, 30.72, 30.72, 30.72, 30.72, 30.72, 30.67, 30.67, 30.67, 30.67, 30.67, 30.27, 30.27, 30.27, 30.27, 30.27, 30.18, 30.18, 30.18, 30.18, 30.18, 30.38, 30.38, 30.38, 30.38, 30.38, 30.4, 30.4, 30.4, 30.4, 30.4, 30.25, 30.25, 30.25, 30.25, 30.25, 30.59, 30.59, 30.59, 30.59, 30.59, 30.62, 30.62, 30.62, 30.62, 30.62, 30.54, 30.54, 30.54, 30.54, 30.54, 30.14, 30.14, 30.14, 30.14, 30.14, 30.31, 30.31, 30.31, 30.31, 30.31, 30.5, 30.5, 30.5, 30.5, 30.5, 30.62, 30.62, 30.62, 30.62, 30.62, 30.66, 30.66, 30.66, 30.66, 30.66, 30.81, 30.81, 30.81, 30.81, 30.81, 30.68, 30.68, 30.68, 30.68, 30.68, 30.65, 30.65, 30.65, 30.65, 30.65, 30.35, 30.35, 30.35, 30.35, 30.35, 29.84, 29.84, 29.84, 29.84, 29.84, 30.02, 30.02, 30.02, 30.02, 30.02, 30.2, 30.2, 30.2, 30.2, 30.2, 30.32, 30.32, 30.32, 30.32, 30.32, 30.41, 30.41, 30.41, 30.41, 30.41, 30.56, 30.56, 30.56, 30.56, 30.56, 30.36, 30.36, 30.36, 30.36, 30.36, 29.91, 29.91, 29.91, 29.91, 29.91, 29.85, 29.85, 29.85, 29.85, 29.85, 29.09, 29.09, 29.09, 29.09, 29.09, 28.92, 28.92, 28.92, 28.92, 28.92, 28.96, 28.96, 28.96, 28.96, 28.96, 29.05, 29.05, 29.05, 29.05, 29.05, 29.05, 29.05, 29.05, 29.05, 29.05, 29.12, 29.12, 29.12, 29.12, 29.12, 29.14, 29.14, 29.14, 29.14, 29.14, 29.07, 29.07, 29.07, 29.07, 29.07, 28.98, 28.98, 28.98, 28.98, 28.98, 28.92, 28.92, 28.92, 28.92, 28.92, 28.96, 28.96, 28.96, 28.96, 28.96, 29.01, 29.01, 29.01, 29.01, 29.01, 29.12, 29.12, 29.12, 29.12, 29.12, 29.19, 29.19, 29.19, 29.19]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 525 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1716897664 --> 1716898290
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.15, 0.15, 0.15, 0.15, 0.15, 0.46, 0.46, 0.46, 0.46, 0.46, 0.08, 0.08, 0.08, 0.08, 0.08, 0.17, 0.17, 0.17, 0.17, 0.17, 0.16, 0.16, 0.16, 0.16, 0.16, 0.1, 0.1, 0.1, 0.1, 0.1, 0.15, 0.15, 0.15, 0.15, 0.15, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.23, 0.23, 0.23, 0.23, 0.23, 0.22, 0.22, 0.22, 0.22, 0.22, 0.14, 0.14, 0.14, 0.14, 0.14, 0.25, 0.25, 0.25, 0.25, 0.25, 0.32, 0.32, 0.32, 0.32, 0.32, 0.45, 0.45, 0.45, 0.45, 0.45, 0.29, 0.29, 0.29, 0.29, 0.29, 0.16, 0.16, 0.16, 0.16, 0.16, 0.15, 0.15, 0.15, 0.15, 0.15, 0.39, 0.39, 0.39, 0.39, 0.39, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.26, 0.26, 0.26, 0.26, 0.26, 0.13, 0.13, 0.13, 0.13, 0.13, 0.17, 0.17, 0.17, 0.17, 0.17, 0.33, 0.33, 0.33, 0.33, 0.33, 0.09, 0.09, 0.09, 0.09, 0.09, 0.12, 0.12, 0.12, 0.12, 0.12, 0.28, 0.28, 0.28, 0.28, 0.28, 0.3, 0.3, 0.3, 0.3, 0.3, 0.13, 0.13, 0.13, 0.13, 0.13, 0.12, 0.12, 0.12, 0.12, 0.12, 0.11, 0.11, 0.11, 0.11, 0.11, 0.18, 0.18, 0.18, 0.18, 0.18, 0.18, 0.18, 0.18, 0.18, 0.18, 0.15, 0.15, 0.15, 0.15, 0.15, 0.29, 0.29, 0.29, 0.29, 0.29, 0.31, 0.31, 0.31, 0.31, 0.31, 0.44, 0.44, 0.44, 0.44, 0.44, 0.2, 0.2, 0.2, 0.2, 0.2, 0.14, 0.14, 0.14, 0.14, 0.14, 0.13, 0.13, 0.13, 0.13, 0.13, 0.12, 0.12, 0.12, 0.12, 0.12, 0.15, 0.15, 0.15, 0.15, 0.15, 0.44, 0.44, 0.44, 0.44, 0.44, 0.6, 0.6, 0.6, 0.6, 0.6, 0.42, 0.42, 0.42, 0.42, 0.42, 0.49, 0.49, 0.49, 0.49, 0.49, 0.22, 0.22, 0.22, 0.22, 0.22, 0.16, 0.16, 0.16, 0.16, 0.16, 0.21, 0.21, 0.21, 0.21, 0.21, 0.17, 0.17, 0.17, 0.17, 0.17, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.28, 0.28, 0.28, 0.28, 0.28, 0.23, 0.23, 0.23, 0.23, 0.23, 0.16, 0.16, 0.16, 0.16, 0.16, 0.3, 0.3, 0.3, 0.3, 0.3, 0.14, 0.14, 0.14, 0.14, 0.14, 0.17, 0.17, 0.17, 0.17, 0.17, 0.08, 0.08, 0.08, 0.08, 0.08, 0.15, 0.15, 0.15, 0.15]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 525 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1716897664 --> 1716898290
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 2.0, 2.0, 2.0, 2.0, 2.0, 4.0, 4.0, 4.0, 4.0, 4.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 1.0, 1.0, 1.0, 1.0, 1.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 2.0, 2.0, 2.0, 2.0, 2.0, 7.0, 7.0, 7.0, 7.0, 7.0, 2.0, 2.0, 2.0, 2.0]
                    

Loading