felixfoertsch/vorleser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert tokenizer to character-by-character, add debug output

Browse Source
This commit is contained in:
Felix Förtsch
2026-02-19 10:39:54 +01:00
parent 4c40e93542
commit 2c975007ca
1 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
VorleserMac/Services/KokoroTokenizer.swift
View File

@@ -26,22 +26,28 @@ struct KokoroTokenizer {
} }
func tokenize(_ phonemes: String, maxTokens: Int = 128) -> (ids: [Int], unknown: [String], scalarCount: Int) { func tokenize(_ phonemes: String, maxTokens: Int = 128) -> (ids: [Int], unknown: [String], scalarCount: Int) {
let cleanedScalars = phonemes.unicodeScalars.filter { $0.properties.generalCategory != .format }
let cleanedView = String.UnicodeScalarView(cleanedScalars)
var ids: [Int] = [] var ids: [Int] = []
var unknown: [String] = [] var unknown: [String] = []
let tokens = phonemes.split(whereSeparator: { $0.isWhitespace }) for scalar in cleanedView {
for token in tokens { let token = String(scalar)
let tokenStr = String(token) if let id = vocab[token] {
if let id = vocab[tokenStr] {
ids.append(id) ids.append(id)
} else { } else {
unknown.append(tokenStr) unknown.append(token)
} }
if ids.count >= maxTokens { if ids.count >= maxTokens {
break break
} }
} }
return (ids, unknown, tokens.count) if !unknown.isEmpty {
print("Tokenizer: \(unknown.count) unknown tokens: \(unknown.prefix(30))")
}
print("Tokenizer: \(ids.count) ids from \(cleanedView.count) scalars, unknown: \(unknown.count)")
return (ids, unknown, cleanedView.count)
} }
} }