mirror of
https://github.com/CJackHwang/ds2api.git
synced 2026-05-02 07:25:26 +08:00
bd41c8a90c
Use go-tiktoken with embedded vocabularies for accurate BPE token counting. CountPromptTokens applies conservative padding so returned context token counts stay slightly above the real value instead of undercounting.
30 lines
784 B
Modula-2
30 lines
784 B
Modula-2
module ds2api
|
|
|
|
go 1.26.0
|
|
|
|
require (
|
|
github.com/andybalholm/brotli v1.2.1
|
|
github.com/go-chi/chi/v5 v5.2.5
|
|
github.com/google/uuid v1.6.0
|
|
github.com/refraction-networking/utls v1.8.2
|
|
github.com/router-for-me/CLIProxyAPI/v6 v6.9.14
|
|
)
|
|
|
|
require (
|
|
github.com/dlclark/regexp2 v1.11.5 // indirect
|
|
github.com/hupe1980/go-tiktoken v0.0.10 // indirect
|
|
)
|
|
|
|
require (
|
|
github.com/klauspost/compress v1.18.5 // indirect
|
|
github.com/sirupsen/logrus v1.9.4 // indirect
|
|
github.com/tidwall/gjson v1.18.0 // indirect
|
|
github.com/tidwall/match v1.2.0 // indirect
|
|
github.com/tidwall/pretty v1.2.1 // indirect
|
|
github.com/tidwall/sjson v1.2.5 // indirect
|
|
golang.org/x/crypto v0.49.0 // indirect
|
|
golang.org/x/net v0.52.0
|
|
golang.org/x/sys v0.42.0 // indirect
|
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
)
|