WideString.h

//========================================================================================
// 
// $File$
// 
// Owner: Nat McCully
// 
// $Author$
// 
// $DateTime$
// 
// $Revision$
// 
// $Change$
// 
// Copyright 1997-2010 Adobe Systems Incorporated. All rights reserved.
// 
// NOTICE: Adobe permits you to use, modify, and distribute this file in accordance 
// with the terms of the Adobe license agreement accompanying it. If you have received
// this file from a source other than Adobe, then your use, modification, or 
// distribution of it requires the prior written permission of Adobe.
// 
// 
// An UTF16-encoded string (16-bit code values).
// 
// SEE USAGE EXAMPLES AT THE BOTTOM OF THIS FILE!
// 
//========================================================================================

#ifndef __WIDESTRING__
#define __WIDESTRING__

#include "PMUtils.h"
#include "K2Iterator.h"
#include "K2TypeTraits.h"
#include "Invariant.h"
#include "UnicodeClass.h"
#include "K2SmartPtr.h"

#include "Trace.h" // /public/includes/

#include "UnicodeSavvyString.h"

class PMString;
class IPMStream;
class TextIterator;


// SEE USAGE EXAMPLES AT THE BOTTOM OF THIS FILE!

// Alias for an unicode code point as defined by the Unicode standard.
// Long story made short: UTF32TextChar was introduced because we wanted to provide
// some type safety. To anybody's indignation, the compilers will happily accept the following code:
//
// typedef uint32 UniCodePoint;
// UniCodePoint u(0xFFFFFFFF);
// char c = u;
//
// Not even a warning is generated. This intention of providing some basic type safety
// is honorable and we respect that. However, we run accros a big problem when trying
// to use the UTF32TextChar in conjuction with boost::tokenizer. The <xstring> header shipped
// with .NET 2003 compiler has a small stirng optimization defined like this:
//
// union _Bxty{
// _Elem _Buf[_BUF_SIZE];
// _Elem *_Ptr;
// } _Bx;
//
// Unfortunately, when instantiating the above template code _Elem template type to UTF32TextChar
// we get a compiler error because you can't put types that have constructors in the unions.
//
// Until we find a solution that will satisfy these two conflicting requirements, we will
// use this typedef for our Unicode code point type.
//
// BEWARE: DON'T MAKE ANY ASSUMPTIONS in your code about the size of a UniCodePoint.
// DO NOT assume that will always be 32-bit and don't cast it around. Just use it as it is.
//
typedef uint32 UniCodePoint;

class WideString;

class WideStringConstUTF32Iter
{
public:

 // Iterator traits: not entirely correct, but work for now :)
 typedef std::bidirectional_iterator_tag iterator_category;
 typedef UniCodePoint value_type;
 typedef std::ptrdiff_t difference_type;
 typedef UTF16TextChar* pointer;
 typedef UTF16TextChar& reference;

 WideStringConstUTF32Iter(const WideString *string, int32 charIndex = 0);

 WideStringConstUTF32Iter(const UTF16TextChar *buffer, bool16 hasSurrogates, int32 numChars) : fCurrent(buffer), fHasSurrogates(hasSurrogates)
 {
#ifdef DEBUG
 fNumChars = numChars;
 fPosition = 0;
#else
 numChars++; //make release happy
#endif
 }

 const UTF16TextChar* PtrAt() const
 { return fCurrent; }

 value_type operator*() const
 {
#ifdef DEBUG 
 ASSERT(fPosition >= 0 && fPosition <= fNumChars);
#endif
 return fHasSurrogates ? surro_operStar() : *fCurrent; 
 }

 WideStringConstUTF32Iter& operator++()
 { if (fHasSurrogates) surro_operPP();
 else ++fCurrent;
#ifdef DEBUG
 ++fPosition;
 ASSERT(fPosition <= fNumChars); // can be one off end
#endif
 return *this; }

 WideStringConstUTF32Iter operator++(int)
 { WideStringConstUTF32Iter tmp(*this); ++(*this); return tmp; }

 WideStringConstUTF32Iter& operator--()
 { if (fHasSurrogates) surro_operMM();
 else --fCurrent;
#ifdef DEBUG
 --fPosition;
 ASSERT(fPosition >= 0);
#endif
 return *this; }

 WideStringConstUTF32Iter operator--(int)
 { WideStringConstUTF32Iter tmp(*this); --(*this); return tmp; }

 WideStringConstUTF32Iter& operator+=(int32 n)
 { if (fHasSurrogates) surro_operPE(n);
 else {
 fCurrent += n;
#ifdef DEBUG
 fPosition += n;
 }
 ASSERT(fPosition <= fNumChars); // can be one off end
#else
 }
#endif
 return *this; }

 WideStringConstUTF32Iter operator+(int32 n) const
 { return WideStringConstUTF32Iter(*this) += n; }

 WideStringConstUTF32Iter& operator-=(int32 n)
 { if (fHasSurrogates) surro_operME(n);
 else {
 fCurrent -= n;
#ifdef DEBUG
 fPosition -= n;
 }
 ASSERT(fPosition >= 0);
#else
 }
#endif
 return *this; }

 WideStringConstUTF32Iter operator-(int32 n) const
 { return WideStringConstUTF32Iter(*this) -= n; }

 int32 operator-(const WideStringConstUTF32Iter& other) const
 { return fHasSurrogates ? surro_operDiff(other) : fCurrent - other.fCurrent; }

 UTF32TextChar operator[](int32 i) const
 { WideStringConstUTF32Iter tmp(*this); tmp += i; return *tmp; }

 friend WideStringConstUTF32Iter operator+(int32 n, const WideStringConstUTF32Iter& rhs)
 { return WideStringConstUTF32Iter(rhs) += n; }

 friend bool operator==(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent == y.fCurrent; }

 friend bool operator!=(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent != y.fCurrent; }

 friend bool operator<(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent < y.fCurrent; }

 friend bool operator<=(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent <= y.fCurrent; }

 friend bool operator>(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent > y.fCurrent; }

 friend bool operator>=(const WideStringConstUTF32Iter& x, const WideStringConstUTF32Iter& y)
 { return x.fCurrent >= y.fCurrent; }

#ifdef DEBUG

 int32 Position() const
 { return fPosition; }

 int32 NumChars() const
 { return fNumChars; }

 int32& NumChars()
 { return fNumChars; }
#endif

private:
 WideStringConstUTF32Iter() {}
 const UTF16TextChar * operator->() const { return fCurrent; }
 UniCodePoint surro_operStar() const;
 void surro_operPP();
 void surro_operPE(int32 n);
 void surro_operMM();
 void surro_operME(int32 n);
 int32 surro_operDiff(const WideStringConstUTF32Iter& other) const;

private:
 const UTF16TextChar *fCurrent;
 bool16 fHasSurrogates;
#ifdef DEBUG
 int32 fNumChars;
 int32 fPosition;
#endif
};



template <typename T>
int32 Strip_If(WideString& string, T f);

class WideString : public UnicodeSavvyString
{
 public:
 typedef object_type data_type;
 typedef UTF16TextChar& reference_raw;
 typedef const UTF16TextChar& const_reference_raw;
 typedef UTF16TextChar value_type_raw;
 typedef std::ptrdiff_t difference_type;
 typedef UTF16TextChar* pointer_raw;
 typedef const UTF16TextChar* const_pointer_raw;

 typedef UTF16TextChar* iterator_raw;
 typedef const UTF16TextChar* const_iterator_raw;
 typedef K2Reverse_iterator<iterator_raw, value_type_raw, difference_type, pointer_raw, reference_raw> reverse_iterator_raw;
 typedef K2Reverse_iterator<const_iterator_raw, value_type_raw, difference_type, const_pointer_raw, const_reference_raw> const_reverse_iterator_raw;

 typedef UTF32TextChar& reference;
 typedef const UTF32TextChar& const_reference;
 typedef UTF32TextChar value_type;
 typedef UTF32TextChar* pointer;
 typedef const UTF32TextChar* const_pointer;

// typedef WideStringUTF32Iter iterator;
 typedef WideStringConstUTF32Iter const_iterator;
 typedef K2Reverse_iterator<const_iterator, value_type, difference_type, const_pointer, const_iterator::value_type> const_reverse_iterator;


 WideString();

 explicit WideString(WideString::const_pointer_raw s, int32 len = kMaxInt32, int32 numChars = -1);

 WideString(const WideString::const_iterator& iter, int32 numChars);

 WideString(const WideString& w);

 WideString(WideString &&other) noexcept = default;

 explicit WideString(ConstCString string, int32 numChars = -1);

 explicit WideString(const PMString& s);

 explicit WideString(const wchar_t* s, int32 len = kMaxInt32); // wchar_t is a distinct type on CW

 template <class IteratorType>
 WideString(IteratorType b, IteratorType e, size_type nCodePoints = 0)
 : UnicodeSavvyString()
 {
 assign(b, e, nCodePoints);
 }

 WideString(adobe::move_from<WideString> other)
 : UnicodeSavvyString(adobe::move_from<UnicodeSavvyString>(other.source))
 {}

 ~WideString();

 void SetString(const WideString& s);
 void SetString(const WideString::const_iterator& iter, int32 numChars);

 void SetCString(ConstCString C, bool16 convertEmbeddedUnicode = kFalse, int32 numChars = -1);

 void SetX16String(WideString::const_pointer_raw x, int32 len = kMaxInt32, int32 numChars = -1); // already 16 bit string
 void SetX16String(const wchar_t* s, int32 len = kMaxInt32, int32 numChars = -1);

 iterator_raw begin_raw();
 const_iterator_raw begin_raw() const;


 iterator_raw end_raw();

 const_iterator_raw end_raw() const;


 reverse_iterator_raw rbegin_raw()
 { TRACE_IF(HasMultiWordUnicode(), "About to hand back non-const iterator on WideString's buffer! Be careful with surrogates!"); return reverse_iterator_raw(end_raw()); }
 const_reverse_iterator_raw rbegin_raw() const
 { return const_reverse_iterator_raw(end_raw()); }

 reverse_iterator_raw rend_raw()
 { TRACE_IF(HasMultiWordUnicode(), "About to hand back non-const iterator on WideString's buffer! Be careful with surrogates!"); return reverse_iterator_raw(begin_raw()); }
 const_reverse_iterator_raw rend_raw() const
 { return const_reverse_iterator_raw(begin_raw()); }


 const_iterator begin() const
 { return const_iterator(this, 0); }

 const_iterator end() const
 { return const_iterator(this, CharCount()); }

 const_reverse_iterator rbegin() const
 { return const_reverse_iterator(end()); }

 const_reverse_iterator rend() const
 { return const_reverse_iterator(begin()); }

 //---------------------------


 int32 UTF16IndexToCodePointIndex(int32 index) const;

 //---------------------------

 bool16 empty() const
 { return fUTF16BufferLength == 0; }

 void Clear()
 { clear(); }

 int32 Length() const
 { return CharCount(); }

 bool16 IsNull() const
 { return empty(); }

 //---------CHAR LEVEL ROUTINES
 UTF32TextChar GetChar(int32 pos) const
 { return GetUTF32TextChar(pos); }
 UTF32TextChar LastChar() const;
 UTF32TextChar FirstChar() const;

 void push_back(WideString::value_type c)
 { Append(c); }

 // these routines append only, no positioning provided (use insert)

 void Append(const WideString &s);

 void Append(const WideString::const_iterator& iter, int32 numChars)
 { Append(iter.PtrAt(), (iter + numChars).PtrAt() - iter.PtrAt(), numChars); }

 void Append(WideString::const_pointer_raw buf, int32 n, int32 numChars = -1);

 WideString& append(WideString::const_pointer_raw s, size_type nCodeValues, size_type nCodePoints = 0)
 {
 UnicodeSavvyString::append(s, nCodeValues, nCodePoints);
 return *this;
 }

 WideString& replace(const WideString::const_iterator& b, const WideString::const_iterator& e, const WideString& s)
 {
 return replace(b, e, s.begin(), s.end());
 }

 WideString& replace(const WideString::const_iterator& b, const WideString::const_iterator& e,
 const WideString::const_iterator& br, const WideString::const_iterator& er);

 WideString& erase(const WideString::const_iterator& b, const WideString::const_iterator& e)
 {
 return replace(b, e, b, b);
 }

 void Append(WideString::value_type c32)
 { UnicodeSavvyString::AppendUTF32TextChar(c32); }

 // these routines insert count character of string s at position.
 void Insert(const WideString &s, int32 position = 0, int32 count = kMaxInt32);
 void Insert(WideString::const_pointer_raw buf, int32 len, int32 pos = 0)
 { if (len > 0) UnicodeSavvyString::InsertUTF16String(buf, len, pos); }
 void Insert(WideString::value_type c, int32 pos = 0)
 { UnicodeSavvyString::InsertUTF32TextChar(c, pos); }

 WideString* Substring(int32 position, int32 count = kMaxInt32) const; // return string from position on, zero = rest

 WideString* GetItem(const WideString& delimiter, const int32 nItem) const; // return the item-th delimited token, one based

 void remove_raw(int32 utf16Pos, int32 utf16Count);
 void RemoveCodePoints(int32 startCodePointIndex, int32 numCodePoints); // assumes legal arguments, doesn't call CountChars()

 int32 IndexOf(const WideString& keyString, int32 position = 0) const; // position of non-overlappin occurence of key string
 int32 IndexOf(WideString::value_type c, int32 position = 0) const;

 bool16 Contains(const WideString& key, int32 pos = 0) const
 { return IndexOf(key, pos) >= 0; }

 void Shrink(bool16 maxshrink = kFalse);

 //----------OPERATORS---------------

 WideString& operator = (const WideString &other)
 {
 UnicodeSavvyString::CopyFrom(other);
 return *this;
 }

 WideString& operator = (WideString &&other) noexcept
 {
 UnicodeSavvyString::move_from(other);
 return *this;
 }

 WideString& operator = (WideString::const_pointer_raw copy)
 {
 return assign(copy, copy + UTF16TextCharLength(copy));
 }

 const UTF32TextChar operator[](int32 index) const
 { return GetChar(index); }

 void SetChar(int32 index, WideString::value_type c32);

 int32 compare(const WideString& s) const;

 template <class IteratorType>
 WideString& assign(IteratorType b, IteratorType e, size_type nCodePoints = 0)
 {
 typedef typename std::iterator_traits<IteratorType>::value_type iterator_value_type;

 // We accept iterators to UTF16 or UTF32 ranges only!
 BOOST_STATIC_ASSERT(sizeof(iterator_value_type)*CHAR_BIT == 16 ||
 sizeof(iterator_value_type)*CHAR_BIT == 32);

 // The code values should be unsigned values
 BOOST_STATIC_ASSERT(!std::numeric_limits<iterator_value_type>::is_signed);

 // Dispatch to the correct impl (UTF16 or UTF32 source)
 EncodingSelector<sizeof(iterator_value_type)*CHAR_BIT> encodingSel;

 WideString::assign_impl(b, e, nCodePoints, encodingSel);
 return *this;
 }

 inline WideString& assign(WideString::const_pointer_raw src, size_type nCodeValues, size_type nCodePoints = 0)
 {
 // Check for negative values
 ASSERT_MSG(nCodeValues != static_cast<size_type>(-1), "-1 is not a valid value for nCodeValues");
 ASSERT_MSG(nCodePoints != static_cast<size_type>(-1), "-1 is not a valid value for nCodePoints");

 return assign(src, src + nCodeValues, nCodePoints);
 }


 bool16 operator >= (const WideString &s) const
 { return compare(s) >= 0; }
 bool16 operator > (const WideString &s) const
 { return compare(s) > 0; }
 bool16 operator <= (const WideString &s) const
 { return compare(s) <= 0; }
 bool16 operator < (const WideString &s) const
 { return compare(s) < 0; }

 bool16 operator == (const WideString &s) const;
 bool16 operator != (const WideString &s) const
 { return !(*this == s); }

 bool16 operator == (WideString::const_pointer_raw b) const;
 bool16 operator != (const UTF16TextChar *b) const
 { return !(*this == b); }

 WideString& operator +=(WideString::value_type c)
 { UnicodeSavvyString::AppendUTF32TextChar(c);
 return *this; }

 WideString& operator +=(const WideString& s)
 { Append(s);
 return *this; }

 //---------------MISC ROUTINES------------------------
 uint32 Hash(void) const;
 void ToLower();
 void ToUpper();

 int32 Strip(WideString::value_type c)
 { return Strip_If(*this, [c](const auto& t){ return t==c;});}

 //----------------------------------------------------
 // conversion routines.
 // *** GetAsSystemString() is now obsolete. In order to convert a WideString
 // *** into a PMString you can either:
 // *** 1. Construct the PMString using a WideString argument.
 // *** 2. Assign the WideString to the PMString using operator=.
 // *** 3. Call GetSystemString().
// PMString* GetAsSystemString(void) const;
 void BuildFromSystemString(const PMString& ss);
 void GetSystemString( PMString *ss ) const;

 //------streaming functions---------
 void ReadWrite(IPMStream* s);

 private:

 // Helper selector for assign() dispatch
 template <unsigned int i> class EncodingSelector {};

 template <class IteratorType>
 inline void assign_impl(IteratorType b, IteratorType e, size_type nCodePoints, EncodingSelector<16>)
 {
 // UnicodeSavvyString handles UTF16s
 UnicodeSavvyString::assign(b, e, nCodePoints);
 }

 template <class IteratorType>
 void assign_impl(IteratorType b, IteratorType e, size_type nCodePoints, EncodingSelector<32>)
 {
 clear();

 // Try to optimize allocations in case the client told us the number of code points.
 if (nCodePoints)
 {
 reserve(nCodePoints);
 }

 // TODO: optimize this - it goes thru an unnecessary UTF32TextChar constructor
 std::copy(b, e, std::back_inserter(*this));
 }


#ifdef DEBUG
 // ---- Data
 mutable InvariantCount fInvariant;
public:
 void Invariant() const;
 // atomic updates and reads
 static bool16 ts_ForceRoundtrip;
private:
 bool16 ParseForMultiWordUnicode() const;
#endif
public:
 static const boost::shared_ptr<WideString> kNil_shared_ptr;
};

inline WideStringConstUTF32Iter::WideStringConstUTF32Iter(const WideString *string, int32 charIndex)
{
 if (string)
 {
 fCurrent = string->GrabUTF16Buffer(nil) + string->CodePointIndexToUTF16Index(charIndex);
 fHasSurrogates = string->HasMultiWordUnicode();
#ifdef DEBUG
 fPosition = charIndex;
 fNumChars = string->CharCount();
#endif
 }
 else
 {
 fCurrent = nil;
 fHasSurrogates = kFalse;
#ifdef DEBUG
 fPosition = 0;
 fNumChars = 0;
#endif
 }
}

inline void swap(WideString& left, WideString& right) noexcept
{
 // Base class swap
 swap(static_cast<UnicodeSavvyString&>(left), static_cast<UnicodeSavvyString&>(right));
}

template <typename T>
int32 Strip_If(WideString& string, T f)
{
 const int32 originalNumChars = string.CharCount();
 int32 num2Strip = 0;

 for (int32 strIndex = originalNumChars - 1; strIndex >= 0; --strIndex)
 {
 if (f(string.GetChar(strIndex)))
 ++num2Strip;
 else if (num2Strip)
 {
 string.RemoveCodePoints(strIndex+1, num2Strip); // start from the next one becaues we just found one NOT to strip
 num2Strip = 0;

 }
 }
 // stripping from the beginning
 if (num2Strip)
 string.RemoveCodePoints(0, num2Strip); // start from the next one becaues we just found one NOT to strip

 return originalNumChars - string.CharCount();
}


inline WideString::const_iterator_raw ToRawIterator(const WideStringConstUTF32Iter& iter)
{ return iter.PtrAt(); }
inline WideString::const_pointer_raw IterToPtr(const WideString::const_iterator_raw& i)
{ return &(*i); }
inline int NumUTF16CodesFromCharacter(const WideString::value_type val)
{ return (val > 0xFFFF ? 2 : 1); }



template <>
inline WideString& WideString::assign<WideString::const_iterator>(WideString::const_iterator b, WideString::const_iterator e, size_type nCodePoints)
{
 UnicodeSavvyString::assign(b.PtrAt(), e.PtrAt(), nCodePoints);
 return *this;
}

// specialize adobe::type_info<> to avoid typeid problems across DLL boundaries
ADOBE_NAME_TYPE_0("widestring:indesign:adobe",WideString);


#endif