This instance will be upgraded to Heptapod 0.23.2 on 2021-08-05 at 11:00 UTC+2 (a few minutes of down time)

Commit 9b140c5b authored by Jesse McGrew's avatar Jesse McGrew
Browse files

Improved abbreviation search using a Generalized Suffix Tree.

This is based on the approach described by Matthew Russotto at <https://intfiction.org/t/highly-optimized-abbreviations-computed-efficiently/48753>.
parent b050094c0140
......@@ -66,6 +66,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Package", "Package", "{3E28
Package\zip\Zip.Packaging.targets = Package\zip\Zip.Packaging.targets
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Zilf.Common.Tests", "test\Zilf.Common.Tests\Zilf.Common.Tests.csproj", "{337F098C-4DD4-420D-825C-4A40DF99AD89}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
CI|Any CPU = CI|Any CPU
......@@ -219,6 +221,18 @@ Global
{55BC8693-4255-4F7C-B69D-734406DE6C88}.Release|Any CPU.ActiveCfg = Debug|Any CPU
{55BC8693-4255-4F7C-B69D-734406DE6C88}.Release|x86.ActiveCfg = Release|Any CPU
{55BC8693-4255-4F7C-B69D-734406DE6C88}.Release|x86.Build.0 = Release|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.CI|Any CPU.ActiveCfg = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.CI|Any CPU.Build.0 = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.CI|x86.ActiveCfg = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.CI|x86.Build.0 = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Debug|Any CPU.Build.0 = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Debug|x86.ActiveCfg = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Debug|x86.Build.0 = Debug|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Release|Any CPU.ActiveCfg = Release|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Release|Any CPU.Build.0 = Release|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Release|x86.ActiveCfg = Release|Any CPU
{337F098C-4DD4-420D-825C-4A40DF99AD89}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
......@@ -242,6 +256,7 @@ Global
{7C9E4970-9997-451B-BCDA-FB869C753F59} = {1E9A9589-73D4-466C-9A3D-D44D058B2359}
{25194AC6-1151-463C-82EA-5D5AD9B56F8D} = {1E9A9589-73D4-466C-9A3D-D44D058B2359}
{3E280263-C59C-4A3B-B9ED-466D22C25333} = {1E9A9589-73D4-466C-9A3D-D44D058B2359}
{337F098C-4DD4-420D-825C-4A40DF99AD89} = {65A2B38C-4851-42E5-900C-DAD903A288F8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {774D42F0-105F-4B51-BC60-F4661B610BAC}
......
......@@ -188,7 +188,8 @@ namespace Zapf.Parsing
if (char.IsDigit((char)c))
digits++;
sb.Append(NextChar());
sb.Append((char)c);
NextChar();
}
var length = sb.Length;
......
/* Copyright 2010-2018 Jesse McGrew
*
* This file is part of ZILF.
*
* ZILF is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ZILF is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with ZILF. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
#if DEBUG_ABBREV
using System.Diagnostics;
#endif
using System.Linq;
using System.Text;
using Zilf.Common.StringEncoding;
namespace Zapf
{
class AbbrevFinder
{
public struct Result
{
public readonly int Score, Count;
public readonly string Text;
public Result(int score, int count, string text)
{
Score = score;
Count = count;
Text = text;
}
}
struct WordRecord
{
public int Savings;
public Horspool Pattern;
}
readonly Dictionary<string, WordRecord> words = new Dictionary<string, WordRecord>();
StringBuilder allText = new StringBuilder();
readonly StringEncoder encoder = new StringEncoder();
/// <summary>
/// Adds some text to the accumulator.
/// </summary>
/// <param name="text">The text to add.</param>
public void AddText(string text)
{
allText.Append(text);
allText.Append('\0');
foreach (string word in FindWords(text))
{
if (!words.ContainsKey(word))
{
var savings = CountSavings(word);
if (savings <= 0)
continue;
words.Add(word, new WordRecord { Savings = savings, Pattern = new Horspool(word) });
}
}
}
/// <summary>
/// Gets the number of characters in the accumulator.
/// </summary>
public int Position => allText.Length;
/// <summary>
/// Rolls the accumulator back to a previous state.
/// </summary>
/// <param name="position">The number of characters to keep.</param>
public void Rollback(int position)
{
if (position < allText.Length && position >= 0)
allText.Length = position;
}
static readonly char[] wordDelimiters = { ' ', '.', ',', ':', ';', '!', '?', '(', ')', '/' };
static IEnumerable<string> FindWords(string text)
{
int wordStart = -1;
bool inWord = false;
for (int i = 0; i < text.Length; i++)
{
char c = text[i];
int wordEnd;
if (inWord)
{
if (Array.IndexOf(wordDelimiters, c) >= 0)
{
inWord = false;
wordEnd = i;
}
else
{
continue;
}
}
else
{
if (c != ' ')
{
inWord = true;
wordStart = i;
}
continue;
}
// found a word
var word = text[wordStart..wordEnd];
bool prev = (wordStart > 0), next = (wordEnd < text.Length);
yield return word;
if (prev)
yield return text[wordStart - 1] + word;
if (prev & next)
yield return text[wordStart - 1] + word + text[wordEnd];
if (next)
yield return word + text[wordEnd];
}
}
int CountSavings(string word)
{
encoder.Encode(word, 0, StringEncoderMode.NoAbbreviations, out int zchars);
return zchars - 2;
}
int CountAppearances(Horspool pattern)
{
#if DEBUG_ABBREV
var stopw = new Stopwatch();
stopw.Start();
#endif
int count = 0, index = -1;
while (true)
{
index = pattern.FindIn(allText, index + 1);
if (index == -1)
break;
count++;
}
#if DEBUG_ABBREV
stopw.Stop();
Console.Error.WriteLine("CountAppearances('{0}') took {1}", pattern.Text, stopw.Elapsed);
#endif
return count;
}
/// <summary>
/// Returns a sequence of abbreviations and clears the previously added text.
/// </summary>
/// <param name="max">The maximum number of abbreviations to return.</param>
/// <returns>A sequence of abbreviations, in descending order of overall savings.</returns>
public IEnumerable<Result> GetResults(int max)
{
try
{
if (max < 1)
yield break;
int numResults = 0;
while (numResults < max)
{
#if DEBUG_ABBREV
var stopw = new Stopwatch();
Console.Error.WriteLine("Querying {0} words in {1} chars of text", words.Count, allText.Length);
stopw.Start();
#endif
var query =
from p in words.AsParallel().WithExecutionMode(ParallelExecutionMode.ForceParallelism)
let count = CountAppearances(p.Value.Pattern)
let overallSavings = (count - 1) * p.Value.Savings - 2
orderby overallSavings descending
select new
{
Savings = overallSavings,
Count = count,
p.Value.Pattern
};
var queryResults = query.ToList();
#if DEBUG_ABBREV
stopw.Stop();
Console.Error.WriteLine("Query time: {0}", stopw.Elapsed);
#endif
foreach (var qr in queryResults)
{
if (qr.Savings <= 0)
{
words.Remove(qr.Pattern.Text);
}
}
if (words.Count == 0 || queryResults.Count == 0)
break;
var r = queryResults[0];
string word = r.Pattern.Text;
yield return new Result(r.Savings, r.Count, word);
numResults++;
if (numResults >= max)
yield break;
int idx, lastIdx = 0;
while ((idx = r.Pattern.FindIn(allText, lastIdx)) >= 0)
{
allText.Remove(idx, word.Length);
allText.Insert(idx, '\0');
lastIdx = idx + 1;
}
var newText = new StringBuilder(allText.Length);
newText.Append(allText);
allText = newText;
words.Remove(word);
}
}
finally
{
words.Clear();
allText.Length = 0;
allText.Capacity = 0;
}
}
}
}
......@@ -244,7 +244,7 @@ namespace Zapf
MaybeProcessEscapeChars(ref str);
var zstr = StringEncoder.Encode(str, mode);
if (FinalPass && AbbreviateMode)
if (FinalPass && AbbreviateMode && mode != StringEncoderMode.NoAbbreviations)
AbbrevFinder.AddText(str);
if (withLength)
......
......@@ -16,6 +16,10 @@
<Link>Zilf.licenseheader</Link>
</None>
</ItemGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
<DebugType>full</DebugType>
<DebugSymbols>true</DebugSymbols>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Zapf.Parsing\Zapf.Parsing.csproj" />
<ProjectReference Include="..\Zilf.Common\Zilf.Common.csproj" />
......
/* Copyright 2010-2020 Jesse McGrew
*
* This file is part of ZILF.
*
* ZILF is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ZILF is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with ZILF. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
namespace Zilf.Common.StringEncoding
{
public sealed class AbbrevFinder
{
public struct Result
{
public readonly int Score, Count;
public readonly string Text;
public Result(int score, int count, string text)
{
Score = score;
Count = count;
Text = text;
}
}
readonly List<string> allTexts = new List<string>();
readonly StringEncoder encoder = new StringEncoder();
/// <summary>
/// Adds some text to the accumulator.
/// </summary>
/// <param name="text">The text to add.</param>
public void AddText(string text)
{
allTexts.Add(text);
}
/// <summary>
/// Gets the number of texts in the accumulator.
/// </summary>
public int Position => allTexts.Count;
/// <summary>
/// Rolls the accumulator back to a previous state.
/// </summary>
/// <param name="position">The number of texts to keep.</param>
public void Rollback(int position)
{
if (position < allTexts.Count && position >= 0)
allTexts.RemoveRange(position, allTexts.Count - position);
}
/// <summary>
/// Returns a sequence of abbreviations and clears the previously added text.
/// </summary>
/// <param name="max">The maximum number of abbreviations to return.</param>
/// <returns>A sequence of abbreviations, in descending order of overall savings.</returns>
public IEnumerable<Result> GetResults(int max)
{
Console.Error.WriteLine("Indexing {0} strings", allTexts.Count);
var outerStopw = new Stopwatch();
outerStopw.Start();
var isc = new IndexedStringCollection(allTexts);
var charsetMap = encoder.GetCharsetMap();
int CostFunction(ReadOnlySpan<char> s)
{
int result = 0;
foreach (var c in s)
{
result += charsetMap.GetValueOrDefault(c) switch
{
// characters in alphabet 0 cost one Z-char each
0 => 1,
// characters in alphabet 1 or 2 cost two Z-chars each
1 or 2 => 2,
// characters in no alphabet cost four Z-chars each
_ => 4,
};
}
return result;
}
int EvaluationFunction(int cost, int count, ReadOnlySpan<char> word)
{
var savings = cost - 2;
return (count - 1) * savings - 2;
}
while (max > 0)
{
#if DEBUG_ABBREV
var stopw = new Stopwatch();
Console.Error.WriteLine("Finding up to {0} abbreviations...", max);
stopw.Start();
#endif
int BATCH_SIZE = 1;
var results = isc.FindBestSubstrings(BATCH_SIZE, CostFunction, EvaluationFunction).ToList();
#if DEBUG_ABBREV
stopw.Stop();
Console.Error.WriteLine("FindBestSubstrings returned {0} results in {1}", results.Count, stopw.Elapsed);
#endif
if (results.Count == 0 || results[0].score < 1)
yield break;
var scores = results.ToDictionary(r => r.substring, r => r.score);
var indexedResults = new IndexedStringCollection(from p in scores where p.Value > 0 select p.Key);
indexedResults.RemoveOverlapping((a, b) =>
{
// keep the higher scoring one
var scoreDiff = scores[a] - scores[b];
if (scoreDiff != 0)
return scoreDiff;
// ...or the shorter one if the scores are equal
return b.Length - a.Length;
});
foreach (var abbrev in indexedResults)
yield return new Result(scores[abbrev], isc.CountOccurrences(abbrev), abbrev);
foreach (var abbrev in indexedResults)
isc.Split(abbrev);
max -= indexedResults.Count;
#if DEBUG_ABBREV
stopw.Stop();
#endif
}
outerStopw.Stop();
Console.Error.WriteLine("Abbreviation search finished in {0}", outerStopw.Elapsed);
}
}
}
/* Copyright 2010-2018 Jesse McGrew
/* Copyright 2010-2020 Jesse McGrew
*
* This file is part of ZILF.
*
......@@ -16,6 +16,7 @@
* along with ZILF. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Generic;
using System.Text;
......@@ -32,6 +33,9 @@ namespace Zilf.Common.StringEncoding
public Horspool(string needle)
{
if (string.IsNullOrEmpty(needle))
throw new ArgumentException("Search string must not be empty", nameof(needle));
this.needle = needle;
int nlen = needle.Length;
......@@ -44,8 +48,7 @@ namespace Zilf.Common.StringEncoding
public string Text => needle;
/*
public int FindIn(string haystack, int startIndex = 0)
public int FindIn(ReadOnlySpan<char> haystack, int startIndex = 0)
{
int hlen = haystack.Length - startIndex;
int hstart = startIndex;
......@@ -65,7 +68,6 @@ namespace Zilf.Common.StringEncoding
return -1;
}
*/
public int FindIn(StringBuilder haystack, int startIndex = 0)
{
......
/* Copyright 2010-2020 Jesse McGrew
*
* This file is part of ZILF.
*
* ZILF is free software: you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ZILF is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with ZILF. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.InteropServices;
using Zilf.Common.StringEncoding.SuffixTrees;
namespace Zilf.Common.StringEncoding
{
public sealed class IndexedStringCollection : ICollection<string>
{
private readonly List<string> strings;
private SuffixTree<int>? suffixTree;
private int maxStringLength;
public int Count => ((ICollection<string>)strings).Count;
public bool IsReadOnly => false;
public IndexedStringCollection()
{
strings = new List<string>();
}
public IndexedStringCollection(IEnumerable<string> strings)
{
this.strings = strings switch
{
ICollection coll => new List<string>(coll.Count),
ICollection<string> coll => new List<string>(coll.Count),
_ => new List<string>(),
};
foreach (var s in strings)
{
this.strings.Add(s);
maxStringLength = Math.Max(maxStringLength, s.Length);
}
}
[MemberNotNull(nameof(suffixTree))]
private SuffixTree<int> BuildSuffixTree()
{
if (suffixTree == null)
{
suffixTree = new SuffixTree<int>();
for (int i = 0; i < strings.Count; i++)
suffixTree.Add(strings[i], i);
}
return suffixTree;
}
public void Add(string str)
{
suffixTree = null;
strings.Add(str);
maxStringLength = Math.Max(maxStringLength, str.Length);
}
public delegate int EdgeCostFunction(ReadOnlySpan<char> chars);
public delegate int EvaluationFunction(int totalEdgeCost, int occurrences, ReadOnlySpan<char> chars);
public int[,] GetSuffixPrefixOverlaps()
{
int k = strings.Count;
var result = new int[k, k];
var stacks = new Stack<int>[k];
for (int i = 0; i < k; i++)
{