[sword-svn] r188 - trunk/source/data/brkitr
chrislit at crosswire.org
chrislit at crosswire.org
Fri May 8 19:32:20 MST 2009
Author: chrislit
Date: 2009-05-08 19:32:20 -0700 (Fri, 08 May 2009)
New Revision: 188
Added:
trunk/source/data/brkitr/char_th.txt
trunk/source/data/brkitr/el.txt
trunk/source/data/brkitr/sent_el.txt
trunk/source/data/brkitr/th.txt
Modified:
trunk/source/data/brkitr/brkfiles.mk
trunk/source/data/brkitr/char.txt
trunk/source/data/brkitr/en.txt
trunk/source/data/brkitr/en_US.txt
trunk/source/data/brkitr/en_US_POSIX.txt
trunk/source/data/brkitr/ja.txt
trunk/source/data/brkitr/line.txt
trunk/source/data/brkitr/root.txt
trunk/source/data/brkitr/sent.txt
trunk/source/data/brkitr/word.txt
trunk/source/data/brkitr/word_POSIX.txt
trunk/source/data/brkitr/word_ja.txt
Log:
ICU 4.2 update, first pass
Modified: trunk/source/data/brkitr/brkfiles.mk
===================================================================
--- trunk/source/data/brkitr/brkfiles.mk 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/brkfiles.mk 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
-# * Copyright (C) 1998-2008, International Business Machines
+# * Copyright (C) 1998-2009, International Business Machines
# * Corporation and others. All Rights Reserved.
-BRK_RES_CLDR_VERSION = 1.6.1
+BRK_RES_CLDR_VERSION = 1.7
# A list of txt's to build
# Note:
#
@@ -38,9 +38,9 @@
# List of break iterator files (brk).
-BRK_SOURCE = word_POSIX.txt word_ja.txt char.txt word.txt line.txt sent.txt title.txt
+BRK_SOURCE = word_POSIX.txt word_ja.txt sent_el.txt char_th.txt char.txt word.txt line.txt sent.txt title.txt
# Ordinary resources
-BRK_RES_SOURCE = en.txt en_US.txt en_US_POSIX.txt ja.txt
+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt ja.txt th.txt
Modified: trunk/source/data/brkitr/char.txt
===================================================================
--- trunk/source/data/brkitr/char.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/char.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,12 +1,12 @@
#
-# Copyright (C) 2002-2008, International Business Machines Corporation and others.
+# Copyright (C) 2002-2009, International Business Machines Corporation and others.
# All Rights Reserved.
#
# file: char.txt
#
# ICU Character Break Rules, also known as Grapheme Cluster Boundaries
# See Unicode Standard Annex #29.
-# These rules are based on TR29 Revision 12, for Unicode Version 5.1
+# These rules are based on TR29 Revision 13, for Unicode Version 5.1
#
#
Added: trunk/source/data/brkitr/char_th.txt
===================================================================
--- trunk/source/data/brkitr/char_th.txt (rev 0)
+++ trunk/source/data/brkitr/char_th.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2002-2009, International Business Machines Corporation and others.
+# All Rights Reserved.
+#
+# file: char_th.txt
+#
+# ICU Character Break Rules, also known as Grapheme Cluster Boundaries
+# See Unicode Standard Annex #29.
+# These rules are based on TR29 Revision 13, for Unicode Version 5.1
+#
+
+#
+# Character Class Definitions.
+#
+$CR = [\p{Grapheme_Cluster_Break = CR}];
+$LF = [\p{Grapheme_Cluster_Break = LF}];
+$Control = [\p{Grapheme_Cluster_Break = Control}];
+$Extend = [\p{Grapheme_Cluster_Break = Extend} - [\u0E30 \u0E32 \u0E45 \u0EB0 \u0EB2]];
+$SpacingMark = [\p{Grapheme_Cluster_Break = SpacingMark}];
+
+#
+# Korean Syllable Definitions
+#
+$L = [\p{Grapheme_Cluster_Break = L}];
+$V = [\p{Grapheme_Cluster_Break = V}];
+$T = [\p{Grapheme_Cluster_Break = T}];
+
+$LV = [\p{Grapheme_Cluster_Break = LV}];
+$LVT = [\p{Grapheme_Cluster_Break = LVT}];
+
+
+## -------------------------------------------------
+!!chain;
+
+!!forward;
+
+$CR $LF;
+
+$L ($L | $V | $LV | $LVT);
+($LV | $V) ($V | $T);
+($LVT | $T) $T;
+
+[^$Control $CR $LF] $Extend;
+
+[^$Control $CR $LF] $SpacingMark;
+
+
+## -------------------------------------------------
+
+!!reverse;
+$LF $CR;
+($L | $V | $LV | $LVT) $L;
+($V | $T) ($LV | $V);
+$T ($LVT | $T);
+
+$Extend [^$Control $CR $LF];
+$SpacingMark [^$Control $CR $LF];
+
+
+## -------------------------------------------------
+
+!!safe_reverse;
+
+
+## -------------------------------------------------
+
+!!safe_forward;
+
Added: trunk/source/data/brkitr/el.txt
===================================================================
--- trunk/source/data/brkitr/el.txt (rev 0)
+++ trunk/source/data/brkitr/el.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -0,0 +1,14 @@
+// ***************************************************************************
+// *
+// * Copyright (C) 2009 International Business Machines
+// * Corporation and others. All Rights Reserved.
+// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
+// * Source File:<path>/xml/brkitr/el.xml
+// *
+// ***************************************************************************
+el{
+ Version{"1.1"}
+ boundaries{
+ sentence:process(dependency){"sent_el.brk"}
+ }
+}
Modified: trunk/source/data/brkitr/en.txt
===================================================================
--- trunk/source/data/brkitr/en.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/en.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
// ***************************************************************************
// *
-// * Copyright (C) 2008 International Business Machines
+// * Copyright (C) 2009 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
// * Source File:<path>/xml/brkitr/en.xml
Modified: trunk/source/data/brkitr/en_US.txt
===================================================================
--- trunk/source/data/brkitr/en_US.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/en_US.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
// ***************************************************************************
// *
-// * Copyright (C) 2008 International Business Machines
+// * Copyright (C) 2009 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
// * Source File:<path>/xml/brkitr/en_US.xml
Modified: trunk/source/data/brkitr/en_US_POSIX.txt
===================================================================
--- trunk/source/data/brkitr/en_US_POSIX.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/en_US_POSIX.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
// ***************************************************************************
// *
-// * Copyright (C) 2008 International Business Machines
+// * Copyright (C) 2009 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
// * Source File:<path>/xml/brkitr/en_US_POSIX.xml
Modified: trunk/source/data/brkitr/ja.txt
===================================================================
--- trunk/source/data/brkitr/ja.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/ja.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
// ***************************************************************************
// *
-// * Copyright (C) 2008 International Business Machines
+// * Copyright (C) 2009 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
// * Source File:<path>/xml/brkitr/ja.xml
Modified: trunk/source/data/brkitr/line.txt
===================================================================
--- trunk/source/data/brkitr/line.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/line.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,11 +1,11 @@
-# Copyright (c) 2002-2008 International Business Machines Corporation and
+# Copyright (c) 2002-2009 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
-# Unicode Standard Annex #14 Revision 21 for Unicode 5.1
+# Unicode Standard Annex #14 Revision 22 for Unicode 5.1
# http://www.unicode.org/reports/tr14/
@@ -23,7 +23,7 @@
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
-# It is used in the implementation of the incredibly annoying rule LB 10
+# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
@@ -359,7 +359,7 @@
# LB 28 Do not break between alphabetics
#
$ALcm $ALcm;
-$CM+ $ALcm; # The $CM+ is from rule 10, and unattached CM is treated as AL
+$CM+ $ALcm; # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm $ALcm;
Modified: trunk/source/data/brkitr/root.txt
===================================================================
--- trunk/source/data/brkitr/root.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/root.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,6 +1,6 @@
// ***************************************************************************
// *
-// * Copyright (C) 2008 International Business Machines
+// * Copyright (C) 2009 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
// * Source File:<path>/xml/brkitr/root.xml
Modified: trunk/source/data/brkitr/sent.txt
===================================================================
--- trunk/source/data/brkitr/sent.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/sent.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,12 +1,12 @@
#
-# Copyright (C) 2002-2008, International Business Machines Corporation and others.
+# Copyright (C) 2002-2009, International Business Machines Corporation and others.
# All Rights Reserved.
#
# file: sent.txt
#
# ICU Sentence Break Rules
# See Unicode Standard Annex #29.
-# These rules are based on SA 29 version 5.1.0
+# These rules are based on UAX 29 Revision 13 for Unicode Version 5.1.0
#
Added: trunk/source/data/brkitr/sent_el.txt
===================================================================
--- trunk/source/data/brkitr/sent_el.txt (rev 0)
+++ trunk/source/data/brkitr/sent_el.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -0,0 +1,119 @@
+#
+# Copyright (C) 2002-2009, International Business Machines Corporation and others.
+# All Rights Reserved.
+#
+# file: sent_el.txt
+#
+# ICU Sentence Break Rules
+# See Unicode Standard Annex #29.
+# These rules are based on UAX 29 Revision 13 for Unicode Version 5.1.0
+#
+
+
+#
+# Character categories as defined in TR 29
+#
+$CR = [\p{Sentence_Break = CR}];
+$LF = [\p{Sentence_Break = LF}];
+$Extend = [\p{Sentence_Break = Extend}];
+$Sep = [\p{Sentence_Break = Sep}];
+$Format = [\p{Sentence_Break = Format}];
+$Sp = [\p{Sentence_Break = Sp}];
+$Lower = [\p{Sentence_Break = Lower}];
+$Upper = [\p{Sentence_Break = Upper}];
+$OLetter = [\p{Sentence_Break = OLetter}];
+$Numeric = [\p{Sentence_Break = Numeric}];
+$ATerm = [\p{Sentence_Break = ATerm}];
+$SContinue = [\p{Sentence_Break = SContinue}];
+$STerm = [\p{Sentence_Break = STerm} [\u003B \u037E]];
+$Close = [\p{Sentence_Break = Close}];
+
+#
+# Define extended forms of the character classes,
+# incorporate trailing Extend or Format chars.
+# Rules 4 and 5.
+
+$SpEx = $Sp ($Extend | $Format)*;
+$LowerEx = $Lower ($Extend | $Format)*;
+$UpperEx = $Upper ($Extend | $Format)*;
+$OLetterEx = $OLetter ($Extend | $Format)*;
+$NumericEx = $Numeric ($Extend | $Format)*;
+$ATermEx = $ATerm ($Extend | $Format)*;
+$SContinueEx= $SContinue ($Extend | $Format)*;
+$STermEx = $STerm ($Extend | $Format)*;
+$CloseEx = $Close ($Extend | $Format)*;
+
+
+## -------------------------------------------------
+
+!!chain;
+!!forward;
+
+# Rule 3 - break after separators. Keep CR/LF together.
+#
+$CR $LF;
+
+
+# Rule 4 - Break after $Sep.
+# Rule 5 - Ignore $Format and $Extend
+#
+[^$Sep $CR $LF]? ($Extend | $Format)*;
+
+
+# Rule 6
+$ATermEx $NumericEx;
+
+# Rule 7
+$UpperEx $ATermEx $UpperEx;
+
+#Rule 8
+$NotLettersEx = [^$OLetter $Upper $Lower $Sep $CR $LF $ATerm $STerm] ($Extend | $Format)*;
+$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
+
+# Rule 8a
+($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx);
+
+#Rule 9, 10, 11
+($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?;
+
+#Rule 12
+[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .;
+[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep $LF $CR {eof}] | $CR $LF){100};
+
+## -------------------------------------------------
+
+!!reverse;
+
+$SpEx_R = ($Extend | $Format)* $Sp;
+$ATermEx_R = ($Extend | $Format)* $ATerm;
+$STermEx_R = ($Extend | $Format)* $STerm;
+$CloseEx_R = ($Extend | $Format)* $Close;
+
+#
+# Reverse rules.
+# For now, use the old style inexact reverse rules, which are easier
+# to write, but less efficient.
+# TODO: exact reverse rules. It appears that exact reverse rules
+# may require improving support for look-ahead breaks in the
+# builder. Needs more investigation.
+#
+
+[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
+#.*;
+
+# Explanation for this rule:
+#
+# It needs to back over
+# The $Sep at which we probably begin
+# All of the non $Sep chars leading to the preceding $Sep
+# The preceding $Sep, which will be the second one that the rule matches.
+# Any immediately preceding STerm or ATerm sequences. We need to see these
+# to get the correct rule status when moving forwards again.
+#
+# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match
+# the entire string.
+#
+# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be
+# at the beginning of the string at this point, and we don't want to fail.
+# Can only use {eof} once, and it is used later.
+#
Added: trunk/source/data/brkitr/th.txt
===================================================================
--- trunk/source/data/brkitr/th.txt (rev 0)
+++ trunk/source/data/brkitr/th.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -0,0 +1,14 @@
+// ***************************************************************************
+// *
+// * Copyright (C) 2009 International Business Machines
+// * Corporation and others. All Rights Reserved.
+// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
+// * Source File:<path>/xml/brkitr/th.xml
+// *
+// ***************************************************************************
+th{
+ Version{"1.1"}
+ boundaries{
+ grapheme:process(dependency){"char_th.brk"}
+ }
+}
Modified: trunk/source/data/brkitr/word.txt
===================================================================
--- trunk/source/data/brkitr/word.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/word.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,12 +1,12 @@
#
-# Copyright (C) 2002-2008, International Business Machines Corporation
+# Copyright (C) 2002-2009, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word.txt
#
# ICU Word Break Rules
# See Unicode Standard Annex #29.
-# These rules are based on UAX-29 Revision 12 for Unicode 5.1
+# These rules are based on UAX-29 Revision 13 for Unicode 5.1
#
# Note: Updates to word.txt will usually need to be merged into
# word_POSIX.txt and word_ja.txt also.
Modified: trunk/source/data/brkitr/word_POSIX.txt
===================================================================
--- trunk/source/data/brkitr/word_POSIX.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/word_POSIX.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,12 +1,12 @@
#
-# Copyright (C) 2002-2008, International Business Machines Corporation
+# Copyright (C) 2002-2009, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word_POSIX.txt
#
# ICU Word Break Rules, POSIX locale.
# See Unicode Standard Annex #29.
-# These rules are based on UAX-29 Revision 12 for Unicode 5.1
+# These rules are based on UAX-29 Revision 13 for Unicode 5.1
#
# Note: Updates to word.txt will usually need to be merged into
# word_POSIX.txt and word_ja.txt also.
@@ -31,9 +31,9 @@
$Format = [\p{Word_Break = Format}];
$Katakana = [\p{Word_Break = Katakana}];
$ALetter = [\p{Word_Break = ALetter}];
-$MidNumLet = [\p{Word_Break = MidNumLet}];
-$MidLetter = [\p{Word_Break = MidLetter} - [ \:]];
-$MidNum = [\p{Word_Break = MidNum}];
+$MidNumLet = [\p{Word_Break = MidNumLet} - [.]];
+$MidLetter = [\p{Word_Break = MidLetter} - [\:]];
+$MidNum = [\p{Word_Break = MidNum} [.]];
$Numeric = [\p{Word_Break = Numeric}];
$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
Modified: trunk/source/data/brkitr/word_ja.txt
===================================================================
--- trunk/source/data/brkitr/word_ja.txt 2009-05-09 02:32:01 UTC (rev 187)
+++ trunk/source/data/brkitr/word_ja.txt 2009-05-09 02:32:20 UTC (rev 188)
@@ -1,12 +1,12 @@
#
-# Copyright (C) 2002-2008, International Business Machines Corporation
+# Copyright (C) 2002-2009, International Business Machines Corporation
# and others. All Rights Reserved.
#
# file: word_ja.txt
#
# ICU Word Break Rules
# See Unicode Standard Annex #29.
-# These rules are based on UAX-29 Revision 12 for Unicode 5.1
+# These rules are based on UAX-29 Revision 13 for Unicode 5.1
#
# Note: Updates to word.txt will usually need to be merged into
# word_POSIX.txt and word_ja.txt also.
@@ -61,7 +61,7 @@
$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;
$Hiragana = [\p{script=Hiragana}];
-$Ideographic = [\p{Ideographic}];
+$Ideographic = [\p{Ideographic} [\u3005 \u3007 \u303B]];
$HiraganaEx = $Hiragana ($Extend | $Format)*;
$IdeographicEx = $Ideographic ($Extend | $Format)*;
More information about the sword-cvs
mailing list