[sword-svn] r3081 - in trunk: include src/modules/filters
chrislit at crosswire.org
chrislit at crosswire.org
Wed Mar 5 12:52:08 MST 2014
Author: chrislit
Date: 2014-03-05 12:52:08 -0700 (Wed, 05 Mar 2014)
New Revision: 3081
Modified:
trunk/include/utf8nfc.h
trunk/include/utf8scsu.h
trunk/src/modules/filters/unicodertf.cpp
trunk/src/modules/filters/utf16utf8.cpp
trunk/src/modules/filters/utf8html.cpp
trunk/src/modules/filters/utf8latin1.cpp
trunk/src/modules/filters/utf8nfc.cpp
trunk/src/modules/filters/utf8nfkd.cpp
trunk/src/modules/filters/utf8scsu.cpp
trunk/src/modules/filters/utf8utf16.cpp
Log:
cleaned out unneeded #includes from Unicode filters
first pass attempt at UTF-8 to SCSU conversion
Modified: trunk/include/utf8nfc.h
===================================================================
--- trunk/include/utf8nfc.h 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/include/utf8nfc.h 2014-03-05 19:52:08 UTC (rev 3081)
@@ -32,10 +32,11 @@
SWORD_NAMESPACE_START
+/** This filter normalizes UTF-8 encoded text
+ */
class SWDLLEXPORT UTF8NFC : public SWFilter {
private:
UConverter* conv;
- UChar *source, *target;
UErrorCode err;
public:
UTF8NFC();
Modified: trunk/include/utf8scsu.h
===================================================================
--- trunk/include/utf8scsu.h 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/include/utf8scsu.h 2014-03-05 19:52:08 UTC (rev 3081)
@@ -25,13 +25,22 @@
#include <swfilter.h>
+#include <unicode/utypes.h>
+#include <unicode/ucnv.h>
+#include <unicode/uchar.h>
+
SWORD_NAMESPACE_START
/** This filter converts UTF-8 encoded text to SCSU
*/
class SWDLLEXPORT UTF8SCSU : public SWFilter {
+private:
+ UConverter* scsuConv;
+ UConverter* utf8Conv;
+ UErrorCode err;
public:
UTF8SCSU();
+ ~UTF8SCSU();
virtual char processText(SWBuf &text, const SWKey *key = 0, const SWModule *module = 0);
};
Modified: trunk/src/modules/filters/unicodertf.cpp
===================================================================
--- trunk/src/modules/filters/unicodertf.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/unicodertf.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -20,7 +20,6 @@
*
*/
-#include <stdlib.h>
#include <stdio.h>
#include <unicodertf.h>
#include <swbuf.h>
Modified: trunk/src/modules/filters/utf16utf8.cpp
===================================================================
--- trunk/src/modules/filters/utf16utf8.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf16utf8.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -21,9 +21,6 @@
*/
-#include <stdlib.h>
-#include <stdio.h>
-
#include <utf16utf8.h>
#include <swbuf.h>
Modified: trunk/src/modules/filters/utf8html.cpp
===================================================================
--- trunk/src/modules/filters/utf8html.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8html.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -21,7 +21,6 @@
*
*/
-#include <stdlib.h>
#include <stdio.h>
#include <utf8html.h>
#include <swbuf.h>
Modified: trunk/src/modules/filters/utf8latin1.cpp
===================================================================
--- trunk/src/modules/filters/utf8latin1.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8latin1.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -20,9 +20,6 @@
*
*/
-#include <stdlib.h>
-#include <stdio.h>
-
#include <utf8latin1.h>
#include <swbuf.h>
Modified: trunk/src/modules/filters/utf8nfc.cpp
===================================================================
--- trunk/src/modules/filters/utf8nfc.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8nfc.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -23,9 +23,6 @@
#ifdef _ICU_
-#include <stdlib.h>
-
-#include <utilstr.h>
#include <unicode/unistr.h>
#include <unicode/normlzr.h>
#include <unicode/unorm.h>
@@ -48,16 +45,16 @@
if ((unsigned long)key < 2) // hack, we're en(1)/de(0)ciphering
return -1;
- UErrorCode status = U_ZERO_ERROR;
- UnicodeString source(text.getRawData(), text.length(), conv, status);
+ err = U_ZERO_ERROR;
+ UnicodeString source(text.getRawData(), text.length(), conv, err);
UnicodeString target;
- status = U_ZERO_ERROR;
- Normalizer::normalize(source, UNORM_NFC, 0, target, status);
+ err = U_ZERO_ERROR;
+ Normalizer::normalize(source, UNORM_NFC, 0, target, err);
- status = U_ZERO_ERROR;
+ err = U_ZERO_ERROR;
text.setSize(text.size()*2); // potentially, it can grow to 2x the original size
- int32_t len = target.extract(text.getRawData(), text.size(), conv, status);
+ int32_t len = target.extract(text.getRawData(), text.size(), conv, err);
text.setSize(len);
return 0;
Modified: trunk/src/modules/filters/utf8nfkd.cpp
===================================================================
--- trunk/src/modules/filters/utf8nfkd.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8nfkd.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -23,10 +23,6 @@
#ifdef _ICU_
-#include <stdlib.h>
-
-#include <utilstr.h>
-
#include <utf8nfkd.h>
#include <swbuf.h>
Modified: trunk/src/modules/filters/utf8scsu.cpp
===================================================================
--- trunk/src/modules/filters/utf8scsu.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8scsu.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -20,30 +20,42 @@
*
*/
-#include <stdlib.h>
-#include <stdio.h>
+#ifdef _ICU_
-#include <sysdata.h>
#include <utf8scsu.h>
-#include <utilstr.h>
#include <swbuf.h>
-
SWORD_NAMESPACE_START
UTF8SCSU::UTF8SCSU() {
+ // initialize SCSU converter
+ scsuConv = ucnv_open("SCSU", &err);
+
+ // initialize UTF-8 converter
+ utf8Conv = ucnv_open("UTF-8", &err);
}
+UTF8SCSU::~UTF8SCSU() {
+ ucnv_close(scsuConv);
+ ucnv_close(utf8Conv);
+}
char UTF8SCSU::processText(SWBuf &text, const SWKey *key, const SWModule *module) {
- const unsigned char *from;
- SWBuf orig = text;
+ if ((unsigned long)key < 2) // hack, we're en(1)/de(0)ciphering
+ return -1;
- from = (const unsigned char *)orig.c_str();
+ err = U_ZERO_ERROR;
+ UnicodeString utf16Text(text.getRawData(), text.length(), utf8Conv, err);
+ err = U_ZERO_ERROR;
+ int32_t len = utf16Text.extract(text.getRawData(), text.size(), scsuConv, err);
+ if (len > text.size()+1) {
+ text.setSize(len+1);
+ int32_t len = utf16Text.extract(text.getRawData(), text.size(), scsuConv, err);
+ }
-
return 0;
}
SWORD_NAMESPACE_END
+#endif
Modified: trunk/src/modules/filters/utf8utf16.cpp
===================================================================
--- trunk/src/modules/filters/utf8utf16.cpp 2014-03-05 06:48:26 UTC (rev 3080)
+++ trunk/src/modules/filters/utf8utf16.cpp 2014-03-05 19:52:08 UTC (rev 3081)
@@ -20,10 +20,7 @@
*
*/
-#include <stdlib.h>
-#include <stdio.h>
-#include <sysdata.h>
#include <utf8utf16.h>
#include <utilstr.h>
#include <swbuf.h>
@@ -68,7 +65,6 @@
text.setSize(text.size()-2);
return 0;
-
}
SWORD_NAMESPACE_END
More information about the sword-cvs
mailing list