关注点:
- 不可见的标签(脚本、样式)
- 块级标签
- 内嵌标签
- Br 标记
- 可换行的空格(前导、尾随和多个空格)
- Hard spaces
- 实体
代数决策:
plain-text = Process(Plain(html))
Plain(node-s) => Plain(node-0), Plain(node-1), ..., Plain(node-N)
Plain(BR) => BR
Plain(not-visible-element(child-s)) => nil
Plain(block-element(child-s)) => BS, Plain(child-s), BE
Plain(inline-element(child-s)) => Plain(child-s)
Plain(text) => ch-0, ch-1, .., ch-N
Process(symbol-s) => Process(start-line, symbol-s)
Process(start-line, BR, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(start-line, BS, symbol-s) => Process(start-line, symbol-s)
Process(start-line, BE, symbol-s) => Process(start-line, symbol-s)
Process(start-line, hard-space, symbol-s) => Print(' '), Process(not-ws, symbol-s)
Process(start-line, space, symbol-s) => Process(start-line, symbol-s)
Process(start-line, common-symbol, symbol-s) => Print(common-symbol),
Process(not-ws, symbol-s)
Process(not-ws, BR|BS|BE, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(not-ws, hard-space, symbol-s) => Print(' '), Process(not-ws, symbol-s)
Process(not-ws, space, symbol-s) => Process(ws, symbol-s)
Process(not-ws, common-symbol, symbol-s) => Process(ws, symbol-s)
Process(ws, BR|BS|BE, symbol-s) => Print('\n'), Process(start-line, symbol-s)
Process(ws, hard-space, symbol-s) => Print(' '), Print(' '),
Process(not-ws, symbol-s)
Process(ws, space, symbol-s) => Process(ws, symbol-s)
Process(ws, common-symbol, symbol-s) => Print(' '), Print(common-symbol),
Process(not-ws, symbol-s)
针对 HtmlAgilityPack 和 System.Xml.Linq 的 C# 决策:
//HtmlAgilityPack part
public static string ToPlainText(this HtmlAgilityPack.HtmlDocument doc)
{
var builder = new System.Text.StringBuilder();
var state = ToPlainTextState.StartLine;
Plain(builder, ref state, new[]{doc.DocumentNode});
return builder.ToString();
}
static void Plain(StringBuilder builder, ref ToPlainTextState state, IEnumerable<HtmlAgilityPack.HtmlNode> nodes)
{
foreach (var node in nodes)
{
if (node is HtmlAgilityPack.HtmlTextNode)
{
var text = (HtmlAgilityPack.HtmlTextNode)node;
Process(builder, ref state, HtmlAgilityPack.HtmlEntity.DeEntitize(text.Text).ToCharArray());
}
else
{
var tag = node.Name.ToLower();
if (tag == "br")
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
else if (NonVisibleTags.Contains(tag))
{
}
else if (InlineTags.Contains(tag))
{
Plain(builder, ref state, node.ChildNodes);
}
else
{
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
Plain(builder, ref state, node.ChildNodes);
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
}
}
}
}
//System.Xml.Linq part
public static string ToPlainText(this IEnumerable<XNode> nodes)
{
var builder = new System.Text.StringBuilder();
var state = ToPlainTextState.StartLine;
Plain(builder, ref state, nodes);
return builder.ToString();
}
static void Plain(StringBuilder builder, ref ToPlainTextState state, IEnumerable<XNode> nodes)
{
foreach (var node in nodes)
{
if (node is XElement)
{
var element = (XElement)node;
var tag = element.Name.LocalName.ToLower();
if (tag == "br")
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
else if (NonVisibleTags.Contains(tag))
{
}
else if (InlineTags.Contains(tag))
{
Plain(builder, ref state, element.Nodes());
}
else
{
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
Plain(builder, ref state, element.Nodes());
if (state != ToPlainTextState.StartLine)
{
builder.AppendLine();
state = ToPlainTextState.StartLine;
}
}
}
else if (node is XText)
{
var text = (XText)node;
Process(builder, ref state, text.Value.ToCharArray());
}
}
}
//common part
public static void Process(System.Text.StringBuilder builder, ref ToPlainTextState state, params char[] chars)
{
foreach (var ch in chars)
{
if (char.IsWhiteSpace(ch))
{
if (IsHardSpace(ch))
{
if (state == ToPlainTextState.WhiteSpace)
builder.Append(' ');
builder.Append(' ');
state = ToPlainTextState.NotWhiteSpace;
}
else
{
if (state == ToPlainTextState.NotWhiteSpace)
state = ToPlainTextState.WhiteSpace;
}
}
else
{
if (state == ToPlainTextState.WhiteSpace)
builder.Append(' ');
builder.Append(ch);
state = ToPlainTextState.NotWhiteSpace;
}
}
}
static bool IsHardSpace(char ch)
{
return ch == 0xA0 || ch == 0x2007 || ch == 0x202F;
}
private static readonly HashSet<string> InlineTags = new HashSet<string>
{
//from https://developer.mozilla.org/en-US/docs/Web/HTML/Inline_elemente
"b", "big", "i", "small", "tt", "abbr", "acronym",
"cite", "code", "dfn", "em", "kbd", "strong", "samp",
"var", "a", "bdo", "br", "img", "map", "object", "q",
"script", "span", "sub", "sup", "button", "input", "label",
"select", "textarea"
};
private static readonly HashSet<string> NonVisibleTags = new HashSet<string>
{
"script", "style"
};
public enum ToPlainTextState
{
StartLine = 0,
NotWhiteSpace,
WhiteSpace,
}
}
例子:
// <div> 1 </div> 2 <div> 3 </div>
1
2
3
// <div>1 <br/><br/>  <b> 2 </b> <div> </div><div> </div>  3</div>
1
2
3
// <span>1<style> text </style><i>2</i></span>3
123
//<div>
// <div>
// <div>
// line1
// </div>
// </div>
//</div>
//<div>line2</div>
line1
line2