改进的脏字过滤算法代码,Asp.Net代码库,德仔网

代码语言
代码分类
【Asp.Net】改进的脏字过滤算法
作者:Dezai.CN / 发布于2011/9/13/ 885
<div>/**//// <summary>
/// 读取脏字字典
/// </summary>
/// <returns></returns>
private string ReadDic()
{
string FileName = HttpContext.Current.Server.MapPath("~/App_Data/test.txt");
if (!File.Exists(FileName))
{
return "";
}
else
{
StreamReader sr = new StreamReader(FileName, Encoding.Default);
string words = sr.ReadToEnd();
sr.Close();
return words;
}
}
private string[] ReadFromText()
{
string FileName = HttpContext.Current.Server.MapPath("~/App_Data/test01.txt");
ArrayList list=new ArrayList();
if (File.Exists(FileName))
{
StreamReader sr = new StreamReader(FileName, Encoding.Default);
string s = "";
int i = 0;
while ((s = sr.ReadLine()) != null)
{
if (i == 10000)
{
break;
}
list.Add(s);
i++;
}
}
return (string[])list.ToArray(typeof(string));
}
/**//// <summary>
///newFilter 的摘要说明
/// </summary>
public class newFilter
{
public newFilter()
{
}
private HashSet<string> hash = new HashSet<string>();
private byte[] fastCheck = new byte[char.MaxValue];
private BitArray charCheck = new BitArray(char.MaxValue);
private int maxWordLength = 0;
private int minWordLength = int.MaxValue;
/**//// <summary>
/// 快速查找
/// </summary>
/// <param name="badWords"></param>
/// <param name="tarStr"></param>
/// <returns></returns>
public bool FastDetect(string[] badWords, string tarStr)
{
bool result;
Prepare(badWords);
result = GoDetect(tarStr);
return result;
}
/**//// <summary>
/// 快速查找初始化
/// </summary>
/// <param name="badWords"></param>
private void Prepare(string[] badWords)
{
foreach (string word in badWords)
{
maxWordLength = Math.Max(maxWordLength, word.Length);
minWordLength = Math.Min(minWordLength, word.Length);
for (int i = 0; i < 7 && i < word.Length; i++)
{
fastCheck[word[i]] |= (byte)(1 << i);
}
for (int i = 7; i < word.Length; i++)
{
fastCheck[word[i]] |= 0x80;
}
if (word.Length == 1)
{
charCheck[word[0]] = true;
}
else
{
hash.Add(word);
}
}
}
/**//// <summary>
/// 快速查找主函数
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
private bool GoDetect(string text)
{
int index = 0;
while (index < text.Length)
{
if ((fastCheck[text[index]] & 1) == 0)
{
while (index < text.Length - 1 && (fastCheck[text[++index]] & 1) == 0) ;
}
for (int j = 1; j <= Math.Min(maxWordLength, text.Length - index - 1); j++)
{
if ((fastCheck[text[index + j]] & (1 << Math.Min(j, 7))) == 0)
{
break;
}
else
{
if (j == 1)
{
if (charCheck[text[index]])
{
return true;
}
}
string sub = text.Substring(index, j + 1);
if (hash.Contains(sub))
{
return true;
}
}
}
index++;
}
return false;
}
/**//// <summary>
/// 快速替换
/// </summary>
/// <param name="badWords"></param>
/// <param name="tarStr"></param>
/// <returns></returns>
public string[] FastReplace(string[] badWords, string tarStr)
{
Prepare(badWords);
string[] result = GoReplace(tarStr);
return result;
}
/**//// <summary>
/// 快速替换主函数
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
private string[] GoReplace(string text)
{
ArrayList list = new ArrayList();
int index = 0;
while (index < text.Length)
{
if ((fastCheck[text[index]] & 1) == 0)//如果这个字符不是脏字的首字符
{
while (index < text.Length - 1 && (fastCheck[text[++index]] & 1) == 0) ;//则检测下一个字符
}
for (int j = 1; j <= Math.Min(maxWordLength, text.Length - index - 1); j++)
{
if ((fastCheck[text[index + j]] & (1 << Math.Min(j, 7))) == 0)//如果这个字符不是脏字中某个字符所在的位置，则break
{
break;
}
else
{
if (j == 1)
{
if (charCheck[text[index]])
{
//text = text.Replace(text[index], '*');
if (!list.Contains(text[index].ToString()))
{
list.Add(text[index].ToString());
}
}
}
string sub = text.Substring(index, j + 1);
int flag = j + 1;
if (hash.Contains(sub)) 
{
for (int k = 1; k <= Math.Min(maxWordLength - sub.Length - 1 , text.Length - index - flag); k++)
{
string newSub = text.Substring(index, j + 1 + k);
if (hash.Contains(newSub))
{
flag = j + 1 + k;
}
}
if (flag > j + 1)//表示有更大的匹配
{
sub = text.Substring(index, flag); 
}
//text = text.Replace(sub, "***");
if (!list.Contains(sub))
{
list.Add(sub);
}
} 
}
}
index++;
}
//return text;
return (string[])list.ToArray(typeof(string));
}
}
</div>
试试其它关键字
　脏字过滤算法　
同语言下
可能有用的
Dezai.CN贡献的其它代码(4037)