Skip to content

LT-22607 TonePars use XAmpleParser code for result processing #1002

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AndyBlack wants to merge 3 commits into
base: main
Choose a base branch
from
Open

LT-22607 TonePars use XAmpleParser code for result processing #1002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eec842e
LT-22607 TonePars use XAmpleParser code for result processing
AndyBlack Jul 9, 2026
7251cf8
Add ToneParsInvoker test fix
AndyBlack Jul 9, 2026
0d94b9f
Make XAmpleParser methods used in ToneParsInvoker be static
AndyBlack Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions Src/LexText/ParserCore/XAmpleParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ private void InitXAmpleAddonDataInfo()
/// </summary>
/// <param name="originalName">The original name to be converted</param>
/// <returns>Converted name</returns>
internal static string ConvertNameToUseAnsiCharacters(string originalName)
public static string ConvertNameToUseAnsiCharacters(string originalName)
{
var sb = new StringBuilder();
char[] letters = originalName.ToCharArray();
Expand Down Expand Up @@ -168,7 +168,14 @@ public ParseResult ParseWord(string word)
{
CheckDisposed();

var results = new StringBuilder(m_xample.ParseWord(word));
var results = m_xample.ParseWord(word);
ParseResult result = ProcessParseResults(results, m_cache);

return result;
}

public static ParseResult ProcessParseResults(String results, LcmCache cache)
{
results = results.Replace("DB_REF_HERE", "'0'");
results = results.Replace("<...>", "[...]");
var wordformElem = XElement.Parse(results.ToString());
Expand All @@ -195,7 +202,7 @@ public ParseResult ParseWord(string word)
}

ParseResult result;
using (new WorkerThreadReadHelper(m_cache.ServiceLocator.GetInstance<IWorkerThreadReadHandler>()))
using (new WorkerThreadReadHelper(cache.ServiceLocator.GetInstance<IWorkerThreadReadHandler>()))
{
var analyses = new List<ParseAnalysis>();
foreach (XElement analysisElem in wordformElem.Descendants("WfiAnalysis"))
Expand All @@ -205,7 +212,7 @@ public ParseResult ParseWord(string word)
foreach (XElement morphElem in analysisElem.Descendants("Morph"))
{
ParseMorph morph;
if (!TryCreateParseMorph(m_cache, morphElem, out morph))
if (!TryCreateParseMorph(cache, morphElem, out morph))
{
skip = true;
break;
Expand Down
3 changes: 2 additions & 1 deletion ...ilities/pcpatrflex/DisambiguateInFLExDB/DisambiguateInFLExDBTests/ToneParsInvokerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
// Copyright (c) 2019 SIL International
// Copyright (c) 2019 SIL International
// This software is licensed under the LGPL, version 2.1 or later
// (http://www.gnu.org/licenses/lgpl-2.1.html)

using NUnit.Framework;
using PtxUtils;
using SIL.DisambiguateInFLExDB;
using SIL.FieldWorks.Common.FwUtils;
using SIL.FieldWorks.WordWorks.Parser;
using SIL.LCModel;
using SIL.ToneParsFLEx;
using System;
Expand Down
200 changes: 2 additions & 198 deletions Src/Utilities/pcpatrflex/ToneParsFLExDll/ToneParsInvoker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
using System.Threading.Tasks;
using System.Threading;
using System.Windows.Forms;
using System.Xml.Linq;
using System.Xml;
using System;
using XAmpleManagedWrapper;
Expand Down Expand Up @@ -70,7 +69,7 @@ LcmCache cache
InputFile = inputFile;
DecompSeparationChar = decomp;
Cache = cache;
DatabaseName = ConvertNameToUseAnsiCharacters(cache.ProjectId.Name);
DatabaseName = XAmpleParser.ConvertNameToUseAnsiCharacters(cache.ProjectId.Name);
InitFileNames();
Queue = new IdleQueue { IsPaused = true };
}
Expand All @@ -84,33 +83,6 @@ private void InitFileNames()
ToneParsLogFile = Path.Combine(Path.GetTempPath(), "ToneParsInvoker.log");
}

// following borrowed from SIL.FieldWorks.WordWorks.Parser (ParserCore.dll)
/// <summary>
/// Convert any characters in the name which are higher than 0x00FF to hex.
/// Neither XAmple nor PC-PATR can read a file name containing letters above 0x00FF.
/// </summary>
/// <param name="originalName">The original name to be converted</param>
/// <returns>Converted name</returns>
internal static string ConvertNameToUseAnsiCharacters(string originalName)
{
var sb = new StringBuilder();
char[] letters = originalName.ToCharArray();
foreach (var letter in letters)
{
int value = Convert.ToInt32(letter);
if (value > 255)
{
string hex = value.ToString("X4");
sb.Append(hex);
}
else
{
sb.Append(letter);
}
}
return sb.ToString();
}

private void CreateToneParsBatchFile()
{
// TonePars
Expand Down Expand Up @@ -684,9 +656,7 @@ public void SaveResultsInDatabase()
IWfiWordform thiswf = GetWordformFromString(wordform);
if (thiswf != null)
{
var parseResult = ConvertParserFilerResultXmlToParseResult(
ParserFilerXMLString
);
var parseResult = XAmpleParser.ProcessParseResults(ParserFilerXMLString, Cache);
m_parseFiler.ProcessParse(thiswf, ParserPriority.Low, parseResult);
}
i++;
Expand Down Expand Up @@ -724,171 +694,5 @@ protected void ExecuteIdleQueue(IdleQueue idleQueue)
task.Delegate(task.Parameter);
idleQueue.Clear();
}

//-----------------------
// ConvertParserFilerResultXmlToParseResult(), TryCreateParseMorph(), and ProcessMsaHvo() are from XAmpleParser.cs
// (I renamed ParseWord() to ConvertParserFilerResultXmlToParseResult() to avoid confusion.)
// Ideally, we'd expose them from XAmpleParser.cs
public ParseResult ConvertParserFilerResultXmlToParseResult(string results)
{
//TODO: fix! CheckDisposed();
results = results.Replace("DB_REF_HERE", "'0'");
results = results.Replace("<...>", "[...]");
var wordformElem = XElement.Parse(results.ToString());
string errorMessage = null;
var exceptionElem = wordformElem.Element("Exception");
if (exceptionElem != null)
{
var totalAnalysesValue = (string)exceptionElem.Attribute("totalAnalyses");
switch ((string)exceptionElem.Attribute("code"))
{
case "ReachedMaxAnalyses":
errorMessage = String.Format(
"Maximum permitted analyses ({0}) reached." /*ParserCoreStrings.ksReachedMaxAnalysesAllowed*/
,
totalAnalysesValue
);
break;
case "ReachedMaxBufferSize":
errorMessage = String.Format(
"Maximum internal buffer size ({0}) reached." /*ParserCoreStrings.ksReachedMaxInternalBufferSize*/
,
totalAnalysesValue
);
break;
}
}
else
{
errorMessage = (string)wordformElem.Element("Error");
}

ParseResult result;
using (
new WorkerThreadReadHelper(
Cache.ServiceLocator.GetInstance<IWorkerThreadReadHandler>()
)
)
{
var analyses = new List<ParseAnalysis>();
foreach (XElement analysisElem in wordformElem.Descendants("WfiAnalysis"))
{
var morphs = new List<ParseMorph>();
bool skip = false;
foreach (XElement morphElem in analysisElem.Descendants("Morph"))
{
ParseMorph morph;
if (!TryCreateParseMorph(Cache, morphElem, out morph))
{
skip = true;
break;
}
if (morph != null)
morphs.Add(morph);
}

if (!skip && morphs.Count > 0)
analyses.Add(new ParseAnalysis(morphs));
}
result = new ParseResult(analyses, errorMessage);
}

return result;
}

private static bool TryCreateParseMorph(
LcmCache cache,
XElement morphElem,
out ParseMorph morph
)
{
XElement formElement = morphElem.Element("MoForm");
Debug.Assert(formElement != null);
var formHvo = (string)formElement.Attribute("DbRef");

XElement msiElement = morphElem.Element("MSI");
Debug.Assert(msiElement != null);
var msaHvo = (string)msiElement.Attribute("DbRef");

// Normally, the hvo for MoForm is a MoForm and the hvo for MSI is an MSA
// There are four exceptions, though, when an irregularly inflected form is involved:
// 1. <MoForm DbRef="x"... and x is an hvo for a LexEntryInflType.
// This is one of the null allomorphs we create when building the
// input for the parser in order to still get the Word Grammar to have something in any
// required slots in affix templates. The parser filer can ignore these.
// 2. <MSI DbRef="y"... and y is an hvo for a LexEntryInflType.
// This is one of the null allomorphs we create when building the
// input for the parser in order to still get the Word Grammar to have something in any
// required slots in affix templates. The parser filer can ignore these.
// 3. <MSI DbRef="y"... and y is an hvo for a LexEntry.
// The LexEntry is a variant form for the first set of LexEntryRefs.
// 4. <MSI DbRef="y"... and y is an hvo for a LexEntry followed by a period and an index digit.
// The LexEntry is a variant form and the (non-zero) index indicates
// which set of LexEntryRefs it is for.
ICmObject objForm;
if (String.IsNullOrEmpty(formHvo) ||
!cache.ServiceLocator
.GetInstance<ICmObjectRepository>()
.TryGetObject(int.Parse(formHvo), out objForm)
)
{
morph = null;
return false;
}
var form = objForm as IMoForm;
if (form == null)
{
morph = null;
return true;
}

// Irregulary inflected forms can have a combination MSA hvo: the LexEntry hvo, a period, and an index to the LexEntryRef
Tuple<int, int> msaTuple = ProcessMsaHvo(msaHvo);
ICmObject objMsa;
if (
!cache.ServiceLocator
.GetInstance<ICmObjectRepository>()
.TryGetObject(msaTuple.Item1, out objMsa)
)
{
morph = null;
return false;
}
var msa = objMsa as IMoMorphSynAnalysis;
if (msa != null)
{
morph = new ParseMorph(form, msa);
return true;
}

var msaAsLexEntry = objMsa as ILexEntry;
if (msaAsLexEntry != null)
{
// is an irregularly inflected form
// get the MoStemMsa of its variant
if (msaAsLexEntry.EntryRefsOS.Count > 0)
{
ILexEntryRef lexEntryRef = msaAsLexEntry.EntryRefsOS[msaTuple.Item2];
ILexSense sense = MorphServices.GetMainOrFirstSenseOfVariant(lexEntryRef);
var inflType = lexEntryRef.VariantEntryTypesRS[0] as ILexEntryInflType;
morph = new ParseMorph(form, sense.MorphoSyntaxAnalysisRA, inflType);
return true;
}
}

// if it is anything else, we ignore it
morph = null;
return true;
}

private static Tuple<int, int> ProcessMsaHvo(string msaHvo)
{
string[] msaHvoParts = msaHvo.Split('.');
return Tuple.Create(
int.Parse(msaHvoParts[0]),
msaHvoParts.Length == 2 ? int.Parse(msaHvoParts[1]) : 0
);
}
// -----------------------
}
}
Loading