// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System;
namespace SixLabors.Fonts.Unicode {
///
/// An enumerator for retrieving Grapheme instances from a .
///
/// Implements the Unicode Grapheme Cluster Algorithm. UAX:29
///
///
/// Supports the UAX #29 extended grapheme cluster rule for Indic conjunct sequences
/// (GB9c) using the property.
///
/// Methods are pattern-matched by compiler to allow using foreach pattern.
///
public ref struct SpanGraphemeEnumerator
{
private ReadOnlySpan source;
private readonly TerminalWidthOptions terminalWidthOptions;
private int sourceOffset;
///
/// Initializes a new instance of the struct.
///
/// The buffer to read from.
public SpanGraphemeEnumerator(ReadOnlySpan source)
: this(source, TerminalWidthOptions.Default)
{
}
///
/// Initializes a new instance of the struct.
///
/// The buffer to read from.
/// The terminal width options to apply while enumerating.
public SpanGraphemeEnumerator(ReadOnlySpan source, TerminalWidthOptions terminalWidthOptions)
{
this.source = source;
this.terminalWidthOptions = terminalWidthOptions;
this.sourceOffset = 0;
this.Current = default;
}
///
/// Gets the element in the collection at the current position of the enumerator.
///
public GraphemeCluster Current { get; private set; }
///
/// Returns an enumerator that iterates through the collection.
///
/// An enumerator that iterates through the collection.
public readonly SpanGraphemeEnumerator GetEnumerator() => this;
///
/// Advances the enumerator to the next element of the collection.
///
///
/// if the enumerator was successfully advanced to the next element;
/// if the enumerator has passed the end of the collection.
///
public bool MoveNext()
{
// GB9c is a stateful rule: whether the next consonant can join depends on
// the InCB classes already consumed into the current cluster. Keep that state
// outside Processor so Processor remains a simple UTF-16/code-point reader.
IndicConjunctState indicConjunctState = default;
TerminalWidthState terminalWidthState = new(this.terminalWidthOptions);
int utf16Offset = this.sourceOffset;
// Accept the current scalar into the cluster and advance to the next scalar.
// IMPORTANT: Processor.Current* represents the next scalar not yet included in CharsConsumed.
void ConsumeCurrentAndAdvance(ref Processor p)
{
indicConjunctState.Consume(p.CurrentCodePoint);
terminalWidthState.Consume(p.CurrentCodePoint, p.CurrentType);
p.MoveNext();
}
// Drain trailers per GB9/GB9a, plus GB9c-style Indic conjunct tailoring.
// GB9 and GB9a always keep Extend, ZWJ, and SpacingMark with the preceding
// cluster. GB9c additionally keeps an Indic consonant with the same cluster
// when the cluster so far matches:
// InCB=Consonant [InCB=Extend InCB=Linker]* InCB=Linker [InCB=Extend InCB=Linker]*
// x InCB=Consonant
void DrainTrailersAndIndicConjuncts(ref Processor p)
{
while (true)
{
// rules GB9, GB9a
while (p.CurrentType is GraphemeClusterClass.Extend
or GraphemeClusterClass.ZeroWidthJoiner
or GraphemeClusterClass.SpacingMark)
{
ConsumeCurrentAndAdvance(ref p);
}
// Rule GB9c only fires when the already-consumed cluster has seen
// a consonant and a following linker. Extend values preserve that
// state, so they are consumed above before this check runs.
if (indicConjunctState.CanLinkConsonant
&& CodePoint.GetIndicConjunctBreakClass(p.CurrentCodePoint) == IndicConjunctBreakClass.Consonant)
{
ConsumeCurrentAndAdvance(ref p);
continue;
}
break;
}
}
if (this.source.IsEmpty)
{
return false;
}
// Algorithm given at https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules.
Processor processor = new(this.source);
processor.MoveNext();
// First, consume as many Prepend scalars as we can (rule GB9b).
while (processor.CurrentType == GraphemeClusterClass.Prepend)
{
ConsumeCurrentAndAdvance(ref processor);
}
// Next, make sure we're not about to violate control character restrictions.
// Essentially, if we saw Prepend data, we can't have Control | CR | LF data afterward (rule GB5).
if (processor.CharsConsumed > 0)
{
if (processor.CurrentType is GraphemeClusterClass.Control
or GraphemeClusterClass.CarriageReturn
or GraphemeClusterClass.LineFeed)
{
goto Return;
}
}
// Now begin the main state machine.
GraphemeClusterClass previousClusterBreakType = processor.CurrentType;
ConsumeCurrentAndAdvance(ref processor);
switch (previousClusterBreakType)
{
case GraphemeClusterClass.CarriageReturn:
if (processor.CurrentType != GraphemeClusterClass.LineFeed)
{
goto Return; // rules GB3 & GB4 (only can follow )
}
ConsumeCurrentAndAdvance(ref processor);
goto case GraphemeClusterClass.LineFeed;
case GraphemeClusterClass.Control:
case GraphemeClusterClass.LineFeed:
goto Return; // rule GB4 (no data after Control | LF)
case GraphemeClusterClass.HangulLead:
if (processor.CurrentType == GraphemeClusterClass.HangulLead)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB6 (L x L)
goto case GraphemeClusterClass.HangulLead;
}
else if (processor.CurrentType == GraphemeClusterClass.HangulVowel)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB6 (L x V)
goto case GraphemeClusterClass.HangulVowel;
}
else if (processor.CurrentType == GraphemeClusterClass.HangulLeadVowel)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB6 (L x LV)
goto case GraphemeClusterClass.HangulLeadVowel;
}
else if (processor.CurrentType == GraphemeClusterClass.HangulLeadVowelTail)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB6 (L x LVT)
goto case GraphemeClusterClass.HangulLeadVowelTail;
}
else
{
break;
}
case GraphemeClusterClass.HangulLeadVowel:
case GraphemeClusterClass.HangulVowel:
if (processor.CurrentType == GraphemeClusterClass.HangulVowel)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB7 (LV | V x V)
goto case GraphemeClusterClass.HangulVowel;
}
else if (processor.CurrentType == GraphemeClusterClass.HangulTail)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB7 (LV | V x T)
goto case GraphemeClusterClass.HangulTail;
}
else
{
break;
}
case GraphemeClusterClass.HangulLeadVowelTail:
case GraphemeClusterClass.HangulTail:
if (processor.CurrentType == GraphemeClusterClass.HangulTail)
{
ConsumeCurrentAndAdvance(ref processor); // rule GB8 (LVT | T x T)
goto case GraphemeClusterClass.HangulTail;
}
else
{
break;
}
case GraphemeClusterClass.ExtendedPictographic:
// Attempt processing extended pictographic (rules GB11, GB9).
// First, drain any Extend scalars that might exist
while (processor.CurrentType == GraphemeClusterClass.Extend)
{
ConsumeCurrentAndAdvance(ref processor);
}
// Now see if there's a ZWJ + extended pictograph again.
if (processor.CurrentType != GraphemeClusterClass.ZeroWidthJoiner)
{
break;
}
ConsumeCurrentAndAdvance(ref processor);
if (processor.CurrentType != GraphemeClusterClass.ExtendedPictographic)
{
break;
}
ConsumeCurrentAndAdvance(ref processor);
goto case GraphemeClusterClass.ExtendedPictographic;
case GraphemeClusterClass.RegionalIndicator:
// We've consumed a single RI scalar. Try to consume another (to make it a pair).
if (processor.CurrentType == GraphemeClusterClass.RegionalIndicator)
{
ConsumeCurrentAndAdvance(ref processor);
}
// Standalone RI scalars (or a single pair of RI scalars) can only be followed by trailers.
break; // nothing but trailers after the final RI
default:
break;
}
DrainTrailersAndIndicConjuncts(ref processor);
Return:
terminalWidthState.Complete();
ReadOnlySpan grapheme = this.source[..processor.CharsConsumed];
this.Current = new GraphemeCluster(
grapheme,
utf16Offset,
terminalWidthState.CodePointCount,
terminalWidthState.TerminalCellWidth,
terminalWidthState.Flags,
terminalWidthState.FirstCodePoint);
this.source = this.source[processor.CharsConsumed..];
this.sourceOffset += processor.CharsConsumed;
return true; // rules GB2, GB999
}
///
/// Tracks terminal width metadata for the grapheme cluster currently being enumerated.
///
///
/// This state is updated as each scalar is accepted into the current UAX #29 cluster, so width,
/// flags, and scalar counts are produced without slicing and re-reading the completed cluster.
///
private struct TerminalWidthState
{
private readonly TerminalWidthOptions options;
///
/// Stores the maximum advancing scalar width before cluster-level overrides are applied.
///
private int terminalCellWidth;
///
/// Indicates that control policy must determine the final cluster width.
///
private bool containsControl;
///
/// Indicates that the cluster contains emoji-related data, even when that data is zero-width.
///
private bool containsEmoji;
///
/// Indicates that terminal practice should treat this emoji-shaped cluster as two cells.
///
private bool containsEmojiWideOverride;
///
/// Stores the first scalar's emoji properties for sequence checks that complete later in the cluster.
///
private EmojiProperties firstEmojiProperties;
///
/// Stores the previous scalar's emoji properties so variation selectors can validate their base.
///
private EmojiProperties previousEmojiProperties;
///
/// Indicates that the cluster contains a valid U+FE0F emoji presentation selector.
///
private bool containsEmojiPresentationSelector;
///
/// Indicates that the cluster contains a valid U+FE0E text presentation selector.
///
private bool containsTextPresentationSelector;
///
/// Initializes a new instance of the struct.
///
/// The terminal width options to apply to the current cluster.
public TerminalWidthState(TerminalWidthOptions options)
{
this.options = options;
this.terminalCellWidth = 0;
this.containsControl = false;
this.containsEmoji = false;
this.containsEmojiWideOverride = false;
this.firstEmojiProperties = EmojiProperties.None;
this.previousEmojiProperties = EmojiProperties.None;
this.containsEmojiPresentationSelector = false;
this.containsTextPresentationSelector = false;
this.CodePointCount = 0;
this.FirstCodePoint = CodePoint.ReplacementChar;
this.Flags = GraphemeClusterFlags.AllZeroWidth;
}
///
/// Gets the number of scalar values consumed into the current cluster.
///
public int CodePointCount { get; private set; }
///
/// Gets the first scalar value consumed into the current cluster.
///
public CodePoint FirstCodePoint { get; private set; }
///
/// Gets the flags derived from the scalars consumed into the current cluster.
///
public GraphemeClusterFlags Flags { get; private set; }
///
/// Gets the policy-resolved terminal cell width of the current cluster.
///
public readonly int TerminalCellWidth
{
get
{
if (this.containsControl)
{
return this.options.ControlCharacterWidth switch
{
TerminalControlCharacterWidth.Zero => 0,
TerminalControlCharacterWidth.Narrow => 1,
_ => -1,
};
}
if (this.containsEmojiWideOverride
&& !this.containsTextPresentationSelector
&& this.options.EmojiWidth == TerminalEmojiWidth.Wide)
{
return 2;
}
return this.terminalCellWidth;
}
}
///
/// Adds a scalar value to the current cluster metadata.
///
/// The scalar value accepted into the current cluster.
/// The grapheme break class for .
public void Consume(in CodePoint codePoint, GraphemeClusterClass graphemeClusterClass)
{
EmojiProperties emojiProperties = CodePoint.GetEmojiProperties(codePoint);
if (this.CodePointCount == 0)
{
this.FirstCodePoint = codePoint;
this.firstEmojiProperties = emojiProperties;
}
this.CodePointCount++;
if (codePoint.Value == 0)
{
return;
}
if (CodePoint.IsControl(codePoint))
{
this.containsControl = true;
this.Flags = (this.Flags & ~GraphemeClusterFlags.AllZeroWidth) | GraphemeClusterFlags.ContainsControl;
return;
}
if (CodePoint.IsVariationSelector(codePoint))
{
this.Flags |= GraphemeClusterFlags.ContainsVariationSelector;
// U+FE0F VARIATION SELECTOR-16 requests emoji presentation.
// Only honor it for bases listed by Unicode as emoji-presentation sequence bases.
if (codePoint.Value == 0xFE0F
&& (this.previousEmojiProperties & EmojiProperties.EmojiPresentationSequenceBase) != 0)
{
this.containsEmoji = true;
this.containsEmojiPresentationSelector = true;
this.containsEmojiWideOverride = true;
}
// U+FE0E VARIATION SELECTOR-15 requests text presentation, which suppresses
// the terminal emoji-wide override even when the base is emoji-capable.
else if (codePoint.Value == 0xFE0E
&& (this.previousEmojiProperties & EmojiProperties.TextPresentationSequenceBase) != 0)
{
this.containsEmoji = true;
this.containsTextPresentationSelector = true;
}
this.previousEmojiProperties = emojiProperties;
return;
}
if (graphemeClusterClass == GraphemeClusterClass.ZeroWidthJoiner)
{
this.Flags |= GraphemeClusterFlags.ContainsZwjSequence;
if (this.containsEmoji)
{
this.containsEmojiWideOverride = true;
}
this.previousEmojiProperties = emojiProperties;
return;
}
if ((emojiProperties & EmojiProperties.EmojiModifier) != 0)
{
this.containsEmoji = true;
this.previousEmojiProperties = emojiProperties;
return;
}
if ((emojiProperties & EmojiProperties.Emoji) != 0)
{
this.containsEmoji = true;
this.Flags |= GraphemeClusterFlags.ContainsEmoji;
}
if ((emojiProperties & EmojiProperties.EmojiPresentation) != 0 && !this.containsTextPresentationSelector)
{
this.containsEmojiWideOverride = true;
}
// U+20E3 COMBINING ENCLOSING KEYCAP completes keycap emoji sequences
// such as "#\uFE0F\u20E3" when the cluster started from a valid keycap base.
if (codePoint.Value == 0x20E3
&& this.containsEmojiPresentationSelector
&& (this.firstEmojiProperties & EmojiProperties.EmojiKeycapSequenceBase) != 0)
{
this.containsEmoji = true;
this.containsEmojiWideOverride = true;
}
if (IsZeroWidthGraphemeExtension(graphemeClusterClass))
{
this.previousEmojiProperties = emojiProperties;
return;
}
if (graphemeClusterClass == GraphemeClusterClass.ExtendedPictographic)
{
this.containsEmoji = true;
this.Flags |= GraphemeClusterFlags.ContainsEmoji;
}
else if (graphemeClusterClass == GraphemeClusterClass.RegionalIndicator)
{
this.containsEmoji = true;
this.containsEmojiWideOverride = true;
this.Flags |= GraphemeClusterFlags.ContainsEmoji;
}
int scalarWidth = this.GetScalarWidth(codePoint);
if (scalarWidth == 0)
{
return;
}
this.Flags &= ~GraphemeClusterFlags.AllZeroWidth;
if (scalarWidth == 2)
{
this.Flags |= GraphemeClusterFlags.ContainsWide;
}
if (scalarWidth > this.terminalCellWidth)
{
this.terminalCellWidth = scalarWidth;
}
this.previousEmojiProperties = emojiProperties;
}
///
/// Finalizes metadata that depends on the completed cluster.
///
public void Complete()
{
if (this.CodePointCount == 1)
{
this.Flags |= GraphemeClusterFlags.IsSingleCodePoint;
}
if (this.containsEmoji)
{
this.Flags |= GraphemeClusterFlags.ContainsEmoji;
}
}
///
/// Gets the terminal cell width contribution for a non-zero-width scalar.
///
/// The scalar value to measure.
/// The scalar width after applying East Asian Width and ambiguous-width policy.
private int GetScalarWidth(in CodePoint codePoint)
{
EastAsianWidthClass width = CodePoint.GetEastAsianWidthClass(codePoint);
if (width == EastAsianWidthClass.Ambiguous)
{
this.Flags |= GraphemeClusterFlags.ContainsAmbiguous;
return this.options.AmbiguousWidth == TerminalAmbiguousWidth.Wide ? 2 : 1;
}
return width is EastAsianWidthClass.Fullwidth or EastAsianWidthClass.Wide ? 2 : 1;
}
///
/// Returns a value indicating whether the grapheme break class contributes no terminal advance.
///
/// The grapheme break class to inspect.
/// if the class is zero-width for terminal measurement.
private static bool IsZeroWidthGraphemeExtension(GraphemeClusterClass graphemeClusterClass)
=> graphemeClusterClass is GraphemeClusterClass.Extend
or GraphemeClusterClass.SpacingMark
or GraphemeClusterClass.Prepend;
}
private ref struct Processor
{
private readonly ReadOnlySpan source;
private int charsConsumed;
public Processor(ReadOnlySpan source)
{
this.source = source;
this.CurrentType = GraphemeClusterClass.Any;
this.CurrentCodePoint = CodePoint.ReplacementChar;
this.charsConsumed = 0;
this.CharsConsumed = 0;
}
public GraphemeClusterClass CurrentType { get; private set; }
public CodePoint CurrentCodePoint { get; private set; }
public int CharsConsumed { get; private set; }
public void MoveNext()
{
this.CharsConsumed += this.charsConsumed;
CodePoint codePoint = CodePoint.DecodeFromUtf16At(this.source, this.CharsConsumed, out this.charsConsumed);
this.CurrentCodePoint = codePoint;
this.CurrentType = CodePoint.GetGraphemeClusterClass(codePoint);
}
}
///
/// Tracks the already-consumed part of the UAX #29 GB9c Indic conjunct rule.
///
///
/// GB9c prevents a grapheme break before an Indic consonant when the current cluster already
/// contains an Indic consonant followed by at least one linker, with optional extend/linker
/// code points in between. This state machine consumes the same code points as the main
/// grapheme enumerator and remembers only the minimum information needed for that decision.
///
private struct IndicConjunctState
{
///
/// Indicates that the current cluster contains an InCB=Consonant starter.
///
private bool hasConsonant;
///
/// Indicates that a linker has been consumed after the current consonant starter.
///
private bool hasLinker;
///
/// Gets a value indicating whether GB9c should suppress a break before the next consonant.
///
///
/// This becomes true only after a consonant and a following linker have both been consumed.
/// InCB=Extend values leave the state unchanged, so combining marks can appear
/// between the linker and the next consonant.
///
public readonly bool CanLinkConsonant => this.hasConsonant && this.hasLinker;
///
/// Updates the GB9c state with a code point that has just been consumed into the cluster.
///
/// The consumed code point.
public void Consume(in CodePoint codePoint)
{
switch (CodePoint.GetIndicConjunctBreakClass(codePoint))
{
case IndicConjunctBreakClass.Consonant:
// A consonant starts or restarts the GB9c candidate. It cannot link a
// following consonant until a linker has also been consumed.
this.hasConsonant = true;
this.hasLinker = false;
break;
case IndicConjunctBreakClass.Linker:
// Linkers only matter after a consonant starter. Leading linkers cannot
// create a GB9c sequence by themselves.
if (this.hasConsonant)
{
this.hasLinker = true;
}
break;
case IndicConjunctBreakClass.Extend:
// Extend values are transparent for GB9c and preserve the current candidate.
break;
default:
// Any other class ends the candidate conjunct sequence.
this.hasConsonant = false;
this.hasLinker = false;
break;
}
}
}
}
}