@Ixx,我不认为你的 Token 类很有用。以下是我要做的修改:
import Cocoa
let input = "Hello everyone. This is a sentence."
let options: NSLinguisticTaggerOptions = .OmitOther
let schemes = [NSLinguisticTagSchemeTokenType]
let tagger = NSLinguisticTagger(tagSchemes: schemes, options: Int(options.rawValue))
let range = NSMakeRange(0, (input as NSString).length)
tagger.string = input
var parts: [String] = [] // here we put all the parts including spaces and punctuation signs, such that we can reconstruct sentence at any time
var words: [String] = []
tagger.enumerateTagsInRange(
range,
scheme: NSLinguisticTagSchemeTokenType,
options: options) {
(tag, tokenRange, _, _) in
let part = (input as NSString).substringWithRange(tokenRange)
parts.append(part)
if tag == "Word" {
words.append(part)
}
}
println(parts)
println(words)
--output:--
[Hello, , everyone, ., , This, , is, , a, , sentence, .]
[Hello, everyone, This, is, a, sentence]
或者,为了获得最大的灵活性,您可以将 tokenType 添加到 Token 类并使用 tokenType 属性过滤令牌数组。对于 schemeNSLinguisticTagSchemeTokenType,标记类型是 Word、标点符号、空格或其他——这是 op 需要了解的唯一标记类型。
class Token {
var text: String
var tokenType: String
init(text: String, tokenType: String) {
self.text = text
self.tokenType = tokenType
}
}
let input = "Hello everyone. This is a sentence."
let options: NSLinguisticTaggerOptions = .OmitOther
let schemes = [NSLinguisticTagSchemeTokenType]
let tagger = NSLinguisticTagger(tagSchemes: schemes, options: Int(options.rawValue))
let range = NSMakeRange(0, (input as NSString).length)
tagger.string = input
var parts: [String] = [] // here we put all the parts including spaces and punctuation signs, such that we can reconstruct sentence at any time
//var words: [String] = []
var tokens: [Token] = []
tagger.enumerateTagsInRange(
range,
scheme: NSLinguisticTagSchemeTokenType,
options: options) {
(tag, tokenRange, _, _) in
let part = (input as NSString).substringWithRange(tokenRange)
parts.append(part)
tokens.append(
Token(text: part, tokenType: tag)
)
}
println(parts) //"[Hello, , everyone, ., , This, , is, , a, , sentence, .]"
let words = tokens.filter({$0.tokenType == "Word"})
println(
words.map({$0.text} //[Hello, everyone, This, is, a, sentence]
)